Web console: expose props for S3 (#9432)

* expose props for S3

* added env inputs

* add scarry warning

* use .password

* put the warning front and center

* Update web-console/src/views/load-data-view/load-data-view.tsx

Co-Authored-By: Suneet Saldanha <44787917+suneet-s@users.noreply.github.com>

* let prettier rewrap the text

Co-authored-by: Suneet Saldanha <44787917+suneet-s@users.noreply.github.com>

web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap

@@ -29,6 +29,7 @@ exports[`auto-form snapshot matches snapshot 1`] = `
             autocomplete="off"
             class="bp3-input"
             min="0"
+            placeholder=""
             style="padding-right: 10px;"
             type="text"
             value=""
@@ -194,6 +195,7 @@ exports[`auto-form snapshot matches snapshot 1`] = `
       >
         <input
           class="bp3-input"
+          placeholder=""
           style="padding-right: 10px;"
           type="text"
           value=""
@@ -300,6 +302,7 @@ exports[`auto-form snapshot matches snapshot 1`] = `
     >
       <textarea
         class="bp3-input bp3-fill"
+        placeholder=""
       />
     </div>
   </div>

web-console/src/components/auto-form/auto-form.tsx

@@ -47,7 +47,7 @@ export interface Field<M> {
   defaultValue?: any;
   emptyValue?: any;
   suggestions?: Functor<M, Suggestion[]>;
-  placeholder?: string;
+  placeholder?: Functor<M, string>;
   min?: number;
   zeroMeansUndefined?: boolean;
   disabled?: Functor<M, boolean>;
@@ -56,13 +56,14 @@ export interface Field<M> {
   adjustment?: (model: M) => M;
 }
 
-export interface AutoFormProps<T> {
-  fields: Field<T>[];
-  model: T | undefined;
-  onChange: (newModel: T) => void;
+export interface AutoFormProps<M> {
+  fields: Field<M>[];
+  model: M | undefined;
+  onChange: (newModel: M) => void;
   onFinalize?: () => void;
-  showCustom?: (model: T) => boolean;
+  showCustom?: (model: M) => boolean;
   large?: boolean;
+  globalAdjustment?: (model: M) => M;
 }
 
 export class AutoForm<T extends Record<string, any>> extends React.PureComponent<AutoFormProps<T>> {
@@ -111,11 +112,15 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
       newModel = deepSet(model, field.name, newValue);
     }
 
+    if (field.adjustment) {
+      newModel = field.adjustment(newModel);
+    }
+
     this.modelChange(newModel);
   };
 
   private modelChange = (newModel: T) => {
-    const { fields, onChange, model } = this.props;
+    const { globalAdjustment, fields, onChange, model } = this.props;
 
     // Delete things that are not defined now (but were defined prior to the change)
     for (const someField of fields) {
@@ -127,11 +132,9 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
       }
     }
 
-    // Perform any adjustments if needed
-    for (const someField of fields) {
-      if (someField.adjustment) {
-        newModel = someField.adjustment(newModel);
-      }
+    // Perform any global adjustments if needed
+    if (globalAdjustment) {
+      newModel = globalAdjustment(newModel);
     }
 
     onChange(newModel);
@@ -162,7 +165,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         fill
         large={large}
         disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
-        placeholder={field.placeholder}
+        placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')}
         intent={
           AutoForm.evaluateFunctor(field.required, model, false) && modelValue == null
             ? AutoForm.REQUIRED_INTENT
@@ -203,14 +206,14 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
       <SuggestibleInput
         value={modelValue != null ? modelValue : field.defaultValue || ''}
         onValueChange={v => {
-          if (sanitize) v = sanitize(v);
+          if (sanitize && typeof v === 'string') v = sanitize(v);
           this.fieldChange(field, v);
         }}
         onBlur={() => {
           if (modelValue === '') this.fieldChange(field, undefined);
         }}
         onFinalize={onFinalize}
-        placeholder={field.placeholder}
+        placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')}
         suggestions={AutoForm.evaluateFunctor(field.suggestions, model, undefined)}
         large={large}
         disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
@@ -268,7 +271,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
       <JsonInput
         value={deepGet(model as any, field.name)}
         onChange={(v: any) => this.fieldChange(field, v)}
-        placeholder={field.placeholder}
+        placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')}
       />
     );
   }
@@ -282,7 +285,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         onChange={(v: any) => {
           this.fieldChange(field, v);
         }}
-        placeholder={field.placeholder}
+        placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')}
         large={large}
         disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
         intent={
@@ -304,7 +307,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         onValueChange={(v: any) => {
           this.fieldChange(field, v);
         }}
-        placeholder={field.placeholder}
+        placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')}
       />
     );
   }

web-console/src/components/suggestible-input/suggestible-input.tsx

@@ -35,10 +35,10 @@ export interface SuggestionGroup {
   suggestions: string[];
 }
 
-export type Suggestion = string | SuggestionGroup;
+export type Suggestion = undefined | string | SuggestionGroup;
 
 export interface SuggestibleInputProps extends HTMLInputProps {
-  onValueChange: (newValue: string) => void;
+  onValueChange: (newValue: undefined | string) => void;
   onFinalize?: () => void;
   suggestions?: Suggestion[];
   large?: boolean;
@@ -53,7 +53,7 @@ export class SuggestibleInput extends React.PureComponent<SuggestibleInputProps>
     // this.state = {};
   }
 
-  public handleSuggestionSelect(suggestion: string) {
+  public handleSuggestionSelect(suggestion: undefined | string) {
     const { onValueChange, onFinalize } = this.props;
     onValueChange(suggestion);
     if (onFinalize) onFinalize();
@@ -66,7 +66,15 @@ export class SuggestibleInput extends React.PureComponent<SuggestibleInputProps>
     return (
       <Menu>
         {suggestions.map(suggestion => {
-          if (typeof suggestion === 'string') {
+          if (typeof suggestion === 'undefined') {
+            return (
+              <MenuItem
+                key="__undefined__"
+                text="(none)"
+                onClick={() => this.handleSuggestionSelect(suggestion)}
+              />
+            );
+          } else if (typeof suggestion === 'string') {
             return (
               <MenuItem
                 key={suggestion}

web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap

@@ -107,6 +107,7 @@ exports[`compaction dialog matches snapshot 1`] = `
                     autocomplete="off"
                     class="bp3-input"
                     min="0"
+                    placeholder=""
                     style="padding-right: 10px;"
                     type="text"
                     value="419430400"
@@ -215,6 +216,7 @@ exports[`compaction dialog matches snapshot 1`] = `
               >
                 <input
                   class="bp3-input"
+                  placeholder=""
                   style="padding-right: 10px;"
                   type="text"
                   value="P1D"
@@ -275,6 +277,7 @@ exports[`compaction dialog matches snapshot 1`] = `
                     autocomplete="off"
                     class="bp3-input"
                     min="0"
+                    placeholder=""
                     style="padding-right: 10px;"
                     type="text"
                     value="5000000"
@@ -522,6 +525,7 @@ exports[`compaction dialog matches snapshot 1`] = `
                     autocomplete="off"
                     class="bp3-input"
                     min="0"
+                    placeholder=""
                     style="padding-right: 10px;"
                     type="text"
                     value="25"

web-console/src/utils/ingestion-spec.tsx

@@ -1226,6 +1226,109 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
             </>
           ),
         },
+
+        {
+          name: 'inputSource.properties.accessKeyId.type',
+          label: 'Access key ID type',
+          type: 'string',
+          suggestions: [undefined, 'environment', 'default'],
+          placeholder: '(none)',
+          info: (
+            <>
+              <p>S3 access key type.</p>
+              <p>Setting this will override the default configuration provided in the config.</p>
+              <p>
+                The access key can be pulled from an environment variable or inlined in the
+                ingestion spec (default).
+              </p>
+              <p>
+                Note: Inlining the access key into the ingestion spec is dangerous as it might
+                appear in server log files and can be seen by anyone accessing this console.
+              </p>
+            </>
+          ),
+          adjustment: (ioConfig: IoConfig) => {
+            return deepSet(
+              ioConfig,
+              'inputSource.properties.secretAccessKey.type',
+              deepGet(ioConfig, 'inputSource.properties.accessKeyId.type'),
+            );
+          },
+        },
+        {
+          name: 'inputSource.properties.accessKeyId.variable',
+          label: 'Access key ID environment variable',
+          type: 'string',
+          placeholder: '(environment variable name)',
+          defined: (ioConfig: IoConfig) =>
+            deepGet(ioConfig, 'inputSource.properties.accessKeyId.type') === 'environment',
+          info: <p>The environment variable containing the S3 access key for this S3 bucket.</p>,
+        },
+        {
+          name: 'inputSource.properties.accessKeyId.password',
+          label: 'Access key ID value',
+          type: 'string',
+          placeholder: '(access key)',
+          defined: (ioConfig: IoConfig) =>
+            deepGet(ioConfig, 'inputSource.properties.accessKeyId.type') === 'default',
+          info: (
+            <>
+              <p>S3 access key for this S3 bucket.</p>
+              <p>
+                Note: Inlining the access key into the ingestion spec is dangerous as it might
+                appear in server log files and can be seen by anyone accessing this console.
+              </p>
+            </>
+          ),
+        },
+
+        {
+          name: 'inputSource.properties.secretAccessKey.type',
+          label: 'Secret key type',
+          type: 'string',
+          suggestions: [undefined, 'environment', 'default'],
+          placeholder: '(none)',
+          info: (
+            <>
+              <p>S3 secret key type.</p>
+              <p>Setting this will override the default configuration provided in the config.</p>
+              <p>
+                The secret key can be pulled from an environment variable or inlined in the
+                ingestion spec (default).
+              </p>
+              <p>
+                Note: Inlining the secret key into the ingestion spec is dangerous as it might
+                appear in server log files and can be seen by anyone accessing this console.
+              </p>
+            </>
+          ),
+        },
+        {
+          name: 'inputSource.properties.secretAccessKey.variable',
+          label: 'Secret key value',
+          type: 'string',
+          placeholder: '(environment variable name)',
+          defined: (ioConfig: IoConfig) =>
+            deepGet(ioConfig, 'inputSource.properties.secretAccessKey.type') === 'environment',
+          info: <p>The environment variable containing the S3 secret key for this S3 bucket.</p>,
+        },
+        {
+          name: 'inputSource.properties.secretAccessKey.password',
+          label: 'Secret key value',
+          type: 'string',
+          placeholder: '(secret key)',
+          defined: (ioConfig: IoConfig) =>
+            deepGet(ioConfig, 'inputSource.properties.secretAccessKey.type') === 'default',
+          info: (
+            <>
+              <p>S3 secret key for this S3 bucket.</p>
+              <p>
+                Note: Inlining the access key into the ingestion spec is dangerous as it might
+                appear in server log files and can be seen by anyone accessing this console.
+              </p>
+            </>
+          ),
+        },
       ];
 
     case 'index_parallel:azure':
@@ -2036,7 +2139,6 @@ export function getPartitionRelatedTuningSpecFormFields(
               optimal number of partitions per time chunk and one for generating segments.
             </p>
           ),
-          adjustment: adjustTuningConfig,
         },
         {
           name: 'partitionsSpec.type',

web-console/src/views/load-data-view/load-data-view.tsx

@@ -70,6 +70,7 @@ import { NUMERIC_TIME_FORMATS, possibleDruidFormatForValues } from '../../utils/
 import { updateSchemaWithSample } from '../../utils/druid-type';
 import {
   adjustIngestionSpec,
+  adjustTuningConfig,
   DimensionMode,
   DimensionSpec,
   DimensionsSpec,
@@ -1075,11 +1076,22 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             <LearnMore href={getIngestionDocLink(spec)} />
           </Callout>
           {ingestionComboType ? (
-            <AutoForm
-              fields={getIoConfigFormFields(ingestionComboType)}
-              model={ioConfig}
-              onChange={c => this.updateSpecPreview(deepSet(spec, 'spec.ioConfig', c))}
-            />
+            <>
+              <AutoForm
+                fields={getIoConfigFormFields(ingestionComboType)}
+                model={ioConfig}
+                onChange={c => this.updateSpecPreview(deepSet(spec, 'spec.ioConfig', c))}
+              />
+              {deepGet(spec, 'spec.ioConfig.inputSource.properties.secretAccessKey.password') && (
+                <FormGroup>
+                  <Callout intent={Intent.WARNING}>
+                    This key will be visible to anyone accessing this console and may appear in
+                    server logs. For production scenarios, use of a more secure secret key type is
+                    strongly recommended.
+                  </Callout>
+                </FormGroup>
+              )}
+            </>
           ) : (
             <FormGroup label="IO Config">
               <JsonInput
@@ -2747,6 +2759,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           <AutoForm
             fields={getPartitionRelatedTuningSpecFormFields(getSpecType(spec) || 'index_parallel')}
             model={tuningConfig}
+            globalAdjustment={adjustTuningConfig}
             onChange={t => this.updateSpec(deepSet(spec, 'spec.tuningConfig', t))}
           />
         </div>