From 1cff73f3e0556e1e76074bb6974a4b61db3717e1 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadimon@gmail.com>
Date: Wed, 4 Dec 2019 20:21:07 -0800
Subject: [PATCH] Web console: support new ingest spec format (#8828)

* converter v1

* working v1

* update tests

* update tests

* upgrades

* adjust to new API

* remove hack

* fwd

* step

* neo cache

* fix time selection

* smart reset

* parquest autodetection

* add binaryAsString option

* partitionsSpec

* add ORC support

* ingestSegment -> druid

* remove index tasks

* better min

* load data works

* remove downgrade

* filter on group_id

* fix group_id in test

* update auto form for new props

* add dropBeforeByPeriod rule

* simplify

* prettify json
---
 .../assets/{ingestsegment.png => druid.png}   | Bin
 ...static-google-blobstore.png => google.png} | Bin
 web-console/assets/{static-s3.png => s3.png}  | Bin
 .../components/array-input/array-input.tsx    |  11 +-
 .../src/components/auto-form/auto-form.tsx    |  82 +-
 .../__snapshots__/rule-editor.spec.tsx.snap   |  75 +-
 .../components/rule-editor/rule-editor.scss   |   7 +-
 .../components/rule-editor/rule-editor.tsx    | 110 ++-
 .../suggestible-input/suggestible-input.tsx   |   4 +-
 web-console/src/console-application.tsx       |  12 +-
 .../dialogs/doctor-dialog/doctor-checks.tsx   |  24 +-
 .../__snapshots__/ingestion-spec.spec.ts.snap |  76 ++
 web-console/src/utils/druid-type.ts           |  10 +-
 web-console/src/utils/ingestion-spec.spec.ts  |  99 +++
 web-console/src/utils/ingestion-spec.tsx      | 709 +++++++++++-------
 web-console/src/utils/load-rule.ts            | 143 ++--
 web-console/src/utils/object-change.ts        |  10 +
 web-console/src/utils/sampler.ts              | 415 +++++-----
 web-console/src/utils/spec-utils.ts           |   4 +-
 web-console/src/utils/utils.spec.ts           | 225 +++---
 .../ingestion-view.spec.tsx.snap              |   2 +-
 .../ingestion-view/ingestion-view.spec.tsx    |   2 +-
 .../views/ingestion-view/ingestion-view.tsx   |   4 +-
 .../load-data-view.spec.tsx.snap              |   4 +-
 .../filter-table/filter-table.spec.tsx        |   2 +-
 .../load-data-view/load-data-view.spec.tsx    |   2 +-
 .../views/load-data-view/load-data-view.tsx   | 448 +++++------
 .../parse-data-table.spec.tsx                 |   2 +-
 .../parse-data-table/parse-data-table.tsx     |  16 +-
 .../parse-time-table.spec.tsx                 |   6 +-
 .../schema-table/schema-table.spec.tsx        |   2 +-
 .../transform-table/transform-table.spec.tsx  |   2 +-
 .../src/views/query-view/query-view.tsx       |   2 +-
 .../query-view/run-button/run-button.tsx      |   2 +-
 34 files changed, 1386 insertions(+), 1126 deletions(-)
 rename web-console/assets/{ingestsegment.png => druid.png} (100%)
 rename web-console/assets/{static-google-blobstore.png => google.png} (100%)
 rename web-console/assets/{static-s3.png => s3.png} (100%)
 create mode 100644 web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap
 create mode 100644 web-console/src/utils/ingestion-spec.spec.ts

diff --git a/web-console/assets/ingestsegment.png b/web-console/assets/druid.png
similarity index 100%
rename from web-console/assets/ingestsegment.png
rename to web-console/assets/druid.png
diff --git a/web-console/assets/static-google-blobstore.png b/web-console/assets/google.png
similarity index 100%
rename from web-console/assets/static-google-blobstore.png
rename to web-console/assets/google.png
diff --git a/web-console/assets/static-s3.png b/web-console/assets/s3.png
similarity index 100%
rename from web-console/assets/static-s3.png
rename to web-console/assets/s3.png
diff --git a/web-console/src/components/array-input/array-input.tsx b/web-console/src/components/array-input/array-input.tsx
index 02b0f12beb3..ec82932ccb7 100644
--- a/web-console/src/components/array-input/array-input.tsx
+++ b/web-console/src/components/array-input/array-input.tsx
@@ -23,7 +23,7 @@ import { compact } from '../../utils';
 
 export interface ArrayInputProps {
   className?: string;
-  values: string[];
+  values: string[] | undefined;
   onChange: (newValues: string[] | undefined) => void;
   placeholder?: string;
   large?: boolean;
@@ -40,8 +40,11 @@ export const ArrayInput = React.memo(function ArrayInput(props: ArrayInputProps)
     const stringValue = e.target.value;
     const newValues: string[] = stringValue.split(/[,\s]+/).map((v: string) => v.trim());
     const newValuesFiltered = compact(newValues);
-    if (newValues.length === newValuesFiltered.length) {
-      onChange(stringValue === '' ? undefined : newValuesFiltered);
+    if (stringValue === '') {
+      onChange(undefined);
+      setStringValue(undefined);
+    } else if (newValues.length === newValuesFiltered.length) {
+      onChange(newValuesFiltered);
       setStringValue(undefined);
     } else {
       setStringValue(stringValue);
@@ -51,7 +54,7 @@ export const ArrayInput = React.memo(function ArrayInput(props: ArrayInputProps)
   return (
     <TextArea
       className={className}
-      value={stringValue || props.values.join(', ')}
+      value={stringValue || (props.values || []).join(', ')}
       onChange={handleChange}
       placeholder={placeholder}
       large={large}
diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx
index d8bf9d0f5ca..110bf491daf 100644
--- a/web-console/src/components/auto-form/auto-form.tsx
+++ b/web-console/src/components/auto-form/auto-form.tsx
@@ -25,11 +25,13 @@ import { FormGroupWithInfo } from '../form-group-with-info/form-group-with-info'
 import { IntervalInput } from '../interval-input/interval-input';
 import { JsonInput } from '../json-input/json-input';
 import { PopoverText } from '../popover-text/popover-text';
-import { SuggestibleInput, SuggestionGroup } from '../suggestible-input/suggestible-input';
+import { SuggestibleInput, Suggestion } from '../suggestible-input/suggestible-input';
 
 import './auto-form.scss';
 
-export interface Field<T> {
+export type Functor<M, R> = R | ((model: M) => R);
+
+export interface Field<M> {
   name: string;
   label?: string;
   info?: React.ReactNode;
@@ -43,12 +45,14 @@ export interface Field<T> {
     | 'json'
     | 'interval';
   defaultValue?: any;
-  suggestions?: (string | SuggestionGroup)[];
+  suggestions?: Functor<M, Suggestion[]>;
   placeholder?: string;
   min?: number;
-  disabled?: boolean | ((model: T) => boolean);
-  defined?: boolean | ((model: T) => boolean);
-  required?: boolean | ((model: T) => boolean);
+  zeroMeansUndefined?: boolean;
+  disabled?: Functor<M, boolean>;
+  defined?: Functor<M, boolean>;
+  required?: Functor<M, boolean>;
+  adjustment?: (model: M) => M;
 }
 
 export interface AutoFormProps<T> {
@@ -73,21 +77,16 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
     return newLabel;
   }
 
-  static evaluateFunctor<T>(
-    functor: undefined | boolean | ((model: T) => boolean),
-    model: T | undefined,
-    defaultValue = false,
-  ): boolean {
+  static evaluateFunctor<M, R>(
+    functor: undefined | Functor<M, R>,
+    model: M | undefined,
+    defaultValue: R,
+  ): R {
     if (!model || functor == null) return defaultValue;
-    switch (typeof functor) {
-      case 'boolean':
-        return functor;
-
-      case 'function':
-        return functor(model);
-
-      default:
-        throw new TypeError(`invalid functor`);
+    if (typeof functor === 'function') {
+      return (functor as any)(model);
+    } else {
+      return functor;
     }
   }
 
@@ -109,27 +108,42 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
   };
 
   private modelChange = (newModel: T) => {
-    const { fields, onChange } = this.props;
+    const { fields, onChange, model } = this.props;
 
+    // Delete things that are not defined now (but were defined prior to the change)
     for (const someField of fields) {
-      if (!AutoForm.evaluateFunctor(someField.defined, newModel, true)) {
+      if (
+        !AutoForm.evaluateFunctor(someField.defined, newModel, true) &&
+        AutoForm.evaluateFunctor(someField.defined, model, true)
+      ) {
         newModel = deepDelete(newModel, someField.name);
       }
     }
 
+    // Perform any adjustments if needed
+    for (const someField of fields) {
+      if (someField.adjustment) {
+        newModel = someField.adjustment(newModel);
+      }
+    }
+
     onChange(newModel);
   };
 
   private renderNumberInput(field: Field<T>): JSX.Element {
     const { model, large, onFinalize } = this.props;
 
-    const modelValue = deepGet(model as any, field.name) || field.defaultValue;
+    let modelValue = deepGet(model as any, field.name);
+    if (typeof modelValue !== 'number') modelValue = field.defaultValue;
     return (
       <NumericInput
         value={modelValue}
         onValueChange={(valueAsNumber: number, valueAsString: string) => {
           if (valueAsString === '' || isNaN(valueAsNumber)) return;
-          this.fieldChange(field, valueAsNumber);
+          this.fieldChange(
+            field,
+            valueAsNumber === 0 && field.zeroMeansUndefined ? undefined : valueAsNumber,
+          );
         }}
         onBlur={e => {
           if (e.target.value === '') {
@@ -140,10 +154,10 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         min={field.min || 0}
         fill
         large={large}
-        disabled={AutoForm.evaluateFunctor(field.disabled, model)}
+        disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
         placeholder={field.placeholder}
         intent={
-          AutoForm.evaluateFunctor(field.required, model) && modelValue == null
+          AutoForm.evaluateFunctor(field.required, model, false) && modelValue == null
             ? AutoForm.REQUIRED_INTENT
             : undefined
         }
@@ -169,7 +183,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         majorStepSize={1000000}
         fill
         large={large}
-        disabled={AutoForm.evaluateFunctor(field.disabled, model)}
+        disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
       />
     );
   }
@@ -190,11 +204,11 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         }}
         onFinalize={onFinalize}
         placeholder={field.placeholder}
-        suggestions={field.suggestions}
+        suggestions={AutoForm.evaluateFunctor(field.suggestions, model, undefined)}
         large={large}
-        disabled={AutoForm.evaluateFunctor(field.disabled, model)}
+        disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
         intent={
-          AutoForm.evaluateFunctor(field.required, model) && modelValue == null
+          AutoForm.evaluateFunctor(field.required, model, false) && modelValue == null
             ? AutoForm.REQUIRED_INTENT
             : undefined
         }
@@ -206,9 +220,9 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
     const { model, large, onFinalize } = this.props;
     const modelValue = deepGet(model as any, field.name);
     const shownValue = modelValue == null ? field.defaultValue : modelValue;
-    const disabled = AutoForm.evaluateFunctor(field.disabled, model);
+    const disabled = AutoForm.evaluateFunctor(field.disabled, model, false);
     const intent =
-      AutoForm.evaluateFunctor(field.required, model) && modelValue == null
+      AutoForm.evaluateFunctor(field.required, model, false) && modelValue == null
         ? AutoForm.REQUIRED_INTENT
         : undefined;
 
@@ -263,9 +277,9 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         }}
         placeholder={field.placeholder}
         large={large}
-        disabled={AutoForm.evaluateFunctor(field.disabled, model)}
+        disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
         intent={
-          AutoForm.evaluateFunctor(field.required, model) && modelValue == null
+          AutoForm.evaluateFunctor(field.required, model, false) && modelValue == null
             ? AutoForm.REQUIRED_INTENT
             : undefined
         }
diff --git a/web-console/src/components/rule-editor/__snapshots__/rule-editor.spec.tsx.snap b/web-console/src/components/rule-editor/__snapshots__/rule-editor.spec.tsx.snap
index 75828979676..e1e7c49c98a 100644
--- a/web-console/src/components/rule-editor/__snapshots__/rule-editor.spec.tsx.snap
+++ b/web-console/src/components/rule-editor/__snapshots__/rule-editor.spec.tsx.snap
@@ -90,59 +90,54 @@ exports[`rule editor matches snapshot 1`] = `
               >
                 <select>
                   <option
-                    value="load"
+                    value="loadForever"
                   >
-                    Load
+                    loadForever
                   </option>
                   <option
-                    value="drop"
+                    value="loadByInterval"
                   >
-                    Drop
+                    loadByInterval
                   </option>
                   <option
-                    value="broadcast"
+                    value="loadByPeriod"
                   >
-                    Broadcast
-                  </option>
-                </select>
-                <span
-                  class="bp3-icon bp3-icon-double-caret-vertical"
-                  icon="double-caret-vertical"
-                >
-                  <svg
-                    data-icon="double-caret-vertical"
-                    height="16"
-                    viewBox="0 0 16 16"
-                    width="16"
-                  >
-                    <desc>
-                      double-caret-vertical
-                    </desc>
-                    <path
-                      d="M5 7h6a1.003 1.003 0 00.71-1.71l-3-3C8.53 2.11 8.28 2 8 2s-.53.11-.71.29l-3 3A1.003 1.003 0 005 7zm6 2H5a1.003 1.003 0 00-.71 1.71l3 3c.18.18.43.29.71.29s.53-.11.71-.29l3-3A1.003 1.003 0 0011 9z"
-                      fill-rule="evenodd"
-                    />
-                  </svg>
-                </span>
-              </div>
-              <div
-                class="bp3-html-select"
-              >
-                <select>
-                  <option
-                    value="Forever"
-                  >
-                    forever
+                    loadByPeriod
                   </option>
                   <option
-                    value="ByPeriod"
+                    value="dropForever"
                   >
-                    by period
+                    dropForever
                   </option>
                   <option
-                    value="ByInterval"
+                    value="dropByInterval"
                   >
-                    by interval
+                    dropByInterval
+                  </option>
+                  <option
+                    value="dropByPeriod"
+                  >
+                    dropByPeriod
+                  </option>
+                  <option
+                    value="dropBeforeByPeriod"
+                  >
+                    dropBeforeByPeriod
+                  </option>
+                  <option
+                    value="broadcastForever"
+                  >
+                    broadcastForever
+                  </option>
+                  <option
+                    value="broadcastByInterval"
+                  >
+                    broadcastByInterval
+                  </option>
+                  <option
+                    value="broadcastByPeriod"
+                  >
+                    broadcastByPeriod
                   </option>
                 </select>
                 <span
diff --git a/web-console/src/components/rule-editor/rule-editor.scss b/web-console/src/components/rule-editor/rule-editor.scss
index cdb26c264aa..6e289171c1a 100644
--- a/web-console/src/components/rule-editor/rule-editor.scss
+++ b/web-console/src/components/rule-editor/rule-editor.scss
@@ -31,10 +31,7 @@
     }
   }
 
-  .by-period {
-    display: flex;
-    .bp3-input-group {
-      padding-right: 15px;
-    }
+  .include-future {
+    margin-left: 15px;
   }
 }
diff --git a/web-console/src/components/rule-editor/rule-editor.tsx b/web-console/src/components/rule-editor/rule-editor.tsx
index 4ccc092d5db..f811adcdbef 100644
--- a/web-console/src/components/rule-editor/rule-editor.tsx
+++ b/web-console/src/components/rule-editor/rule-editor.tsx
@@ -49,9 +49,6 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
   const [isOpen, setIsOpen] = useState(true);
   if (!rule) return null;
 
-  const ruleLoadType = RuleUtil.getLoadType(rule);
-  const ruleTimeType = RuleUtil.getTimeType(rule);
-
   function removeTier(key: string) {
     const newTierReplicants = Object.assign({}, rule.tieredReplicants);
     delete newTierReplicants[key];
@@ -72,14 +69,12 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
       }
     }
 
-    onChange(RuleUtil.changeTierReplication(rule, newTierName, 1));
+    onChange(RuleUtil.addTieredReplicant(rule, newTierName, 1));
   }
 
   function renderTiers() {
-    if (RuleUtil.getLoadType(rule) !== 'load') return null;
-
     const tieredReplicants = rule.tieredReplicants;
-    if (!tieredReplicants) return null;
+    if (!tieredReplicants) return;
 
     const ruleTiers = Object.keys(tieredReplicants).sort();
     return ruleTiers.map(tier => {
@@ -92,7 +87,7 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
             value={tieredReplicants[tier]}
             onValueChange={(v: number) => {
               if (isNaN(v)) return;
-              onChange(RuleUtil.changeTierReplication(rule, tier, v));
+              onChange(RuleUtil.addTieredReplicant(rule, tier, v));
             }}
             min={1}
             max={256}
@@ -103,7 +98,9 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
           <HTMLSelect
             fill
             value={tier}
-            onChange={(e: any) => onChange(RuleUtil.changeTier(rule, tier, e.target.value))}
+            onChange={(e: any) =>
+              onChange(RuleUtil.renameTieredReplicants(rule, tier, e.target.value))
+            }
           >
             {tiers
               .filter(t => t === tier || !tieredReplicants[t])
@@ -127,7 +124,7 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
 
   function renderTierAdder() {
     const { rule, tiers } = props;
-    if (Object.keys(rule.tieredReplicants || {}).length >= Object.keys(tiers).length) return null;
+    if (Object.keys(rule.tieredReplicants || {}).length >= Object.keys(tiers).length) return;
 
     return (
       <FormGroup className="right">
@@ -138,18 +135,6 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
     );
   }
 
-  function renderColocatedDataSources() {
-    const { rule, onChange } = props;
-    return (
-      <FormGroup label="Colocated datasources:">
-        <TagInput
-          values={rule.colocatedDataSources || []}
-          onChange={(v: any) => onChange(RuleUtil.changeColocatedDataSources(rule, v))}
-          fill
-        />
-      </FormGroup>
-    );
-  }
   return (
     <div className="rule-editor">
       <div className="title">
@@ -172,52 +157,39 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
           <FormGroup>
             <ControlGroup>
               <HTMLSelect
-                value={ruleLoadType}
+                value={rule.type}
                 onChange={(e: any) =>
-                  onChange(RuleUtil.changeLoadType(rule, e.target.value as any))
+                  onChange(RuleUtil.changeRuleType(rule, e.target.value as any))
                 }
               >
-                <option value="load">Load</option>
-                <option value="drop">Drop</option>
-                <option value="broadcast">Broadcast</option>
+                {RuleUtil.TYPES.map(type => {
+                  return (
+                    <option key={type} value={type}>
+                      {type}
+                    </option>
+                  );
+                })}
               </HTMLSelect>
-              <HTMLSelect
-                value={ruleTimeType}
-                onChange={(e: any) =>
-                  onChange(RuleUtil.changeTimeType(rule, e.target.value as any))
-                }
-              >
-                <option value="Forever">forever</option>
-                <option value="ByPeriod">by period</option>
-                <option value="ByInterval">by interval</option>
-              </HTMLSelect>
-              {ruleTimeType === 'ByPeriod' && (
-                <div className={`by-period`}>
-                  <InputGroup
-                    value={rule.period || ''}
-                    onChange={(e: any) =>
-                      onChange(RuleUtil.changePeriod(rule, e.target.value as any))
-                    }
-                    placeholder="P1D"
-                  />
-                  <Switch
-                    large
-                    checked={rule.includeFuture !== undefined ? rule.includeFuture : true}
-                    label={`Include future`}
-                    onChange={() => {
-                      onChange(
-                        RuleUtil.changeIncludeFuture(
-                          rule,
-                          rule.includeFuture !== undefined
-                            ? (!rule.includeFuture as boolean)
-                            : false,
-                        ),
-                      );
-                    }}
-                  />
-                </div>
+              {RuleUtil.hasPeriod(rule) && (
+                <InputGroup
+                  value={rule.period || ''}
+                  onChange={(e: any) =>
+                    onChange(RuleUtil.changePeriod(rule, e.target.value as any))
+                  }
+                  placeholder="P1D"
+                />
               )}
-              {ruleTimeType === 'ByInterval' && (
+              {RuleUtil.hasIncludeFuture(rule) && (
+                <Switch
+                  className="include-future"
+                  checked={rule.includeFuture || false}
+                  label="Include future"
+                  onChange={() => {
+                    onChange(RuleUtil.changeIncludeFuture(rule, !rule.includeFuture));
+                  }}
+                />
+              )}
+              {RuleUtil.hasInterval(rule) && (
                 <InputGroup
                   value={rule.interval || ''}
                   onChange={(e: any) =>
@@ -228,13 +200,21 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
               )}
             </ControlGroup>
           </FormGroup>
-          {ruleLoadType === 'load' && (
+          {RuleUtil.hasTieredReplicants(rule) && (
             <FormGroup>
               {renderTiers()}
               {renderTierAdder()}
             </FormGroup>
           )}
-          {ruleLoadType === 'broadcast' && <FormGroup>{renderColocatedDataSources()}</FormGroup>}
+          {RuleUtil.hasColocatedDataSources(rule) && (
+            <FormGroup label="Colocated datasources">
+              <TagInput
+                values={rule.colocatedDataSources || []}
+                onChange={(v: any) => onChange(RuleUtil.changeColocatedDataSources(rule, v))}
+                fill
+              />
+            </FormGroup>
+          )}
         </Card>
       </Collapse>
     </div>
diff --git a/web-console/src/components/suggestible-input/suggestible-input.tsx b/web-console/src/components/suggestible-input/suggestible-input.tsx
index b786ee55de0..39d40028690 100644
--- a/web-console/src/components/suggestible-input/suggestible-input.tsx
+++ b/web-console/src/components/suggestible-input/suggestible-input.tsx
@@ -35,10 +35,12 @@ export interface SuggestionGroup {
   suggestions: string[];
 }
 
+export type Suggestion = string | SuggestionGroup;
+
 export interface SuggestibleInputProps extends HTMLInputProps {
   onValueChange: (newValue: string) => void;
   onFinalize?: () => void;
-  suggestions?: (string | SuggestionGroup)[];
+  suggestions?: Suggestion[];
   large?: boolean;
   intent?: Intent;
 }
diff --git a/web-console/src/console-application.tsx b/web-console/src/console-application.tsx
index 193e4ed865e..55c4824ae2c 100644
--- a/web-console/src/console-application.tsx
+++ b/web-console/src/console-application.tsx
@@ -70,6 +70,7 @@ export class ConsoleApplication extends React.PureComponent<
 
   private supervisorId?: string;
   private taskId?: string;
+  private taskGroupId?: string;
   private openDialog?: string;
   private datasource?: string;
   private onlyUnavailable?: boolean;
@@ -109,6 +110,7 @@ export class ConsoleApplication extends React.PureComponent<
   private resetInitialsWithDelay() {
     setTimeout(() => {
       this.taskId = undefined;
+      this.taskGroupId = undefined;
       this.supervisorId = undefined;
       this.openDialog = undefined;
       this.datasource = undefined;
@@ -138,8 +140,8 @@ export class ConsoleApplication extends React.PureComponent<
     this.resetInitialsWithDelay();
   };
 
-  private goToIngestionWithTaskId = (taskId?: string, openDialog?: string) => {
-    this.taskId = taskId;
+  private goToIngestionWithTaskGroupId = (taskGroupId?: string, openDialog?: string) => {
+    this.taskGroupId = taskGroupId;
     if (openDialog) this.openDialog = openDialog;
     window.location.hash = 'ingestion';
     this.resetInitialsWithDelay();
@@ -193,7 +195,7 @@ export class ConsoleApplication extends React.PureComponent<
         initSupervisorId={this.supervisorId}
         initTaskId={this.taskId}
         exampleManifestsUrl={exampleManifestsUrl}
-        goToTask={this.goToIngestionWithTaskId}
+        goToIngestion={this.goToIngestionWithTaskGroupId}
       />,
       'narrow-pad',
     );
@@ -235,7 +237,7 @@ export class ConsoleApplication extends React.PureComponent<
     return this.wrapInViewContainer(
       'ingestion',
       <IngestionView
-        taskId={this.taskId}
+        taskGroupId={this.taskGroupId}
         datasourceId={this.datasource}
         openDialog={this.openDialog}
         goToDatasource={this.goToDatasources}
@@ -254,7 +256,7 @@ export class ConsoleApplication extends React.PureComponent<
       <ServicesView
         middleManager={this.middleManager}
         goToQuery={this.goToQuery}
-        goToTask={this.goToIngestionWithTaskId}
+        goToTask={this.goToIngestionWithTaskGroupId}
         capabilities={capabilities}
       />,
     );
diff --git a/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx b/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx
index 8360e2897f7..e65f078d381 100644
--- a/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx
+++ b/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx
@@ -230,23 +230,21 @@ export const DOCTOR_CHECKS: DoctorCheck[] = [
       try {
         testSampledData = await postToSampler(
           {
-            type: 'index',
+            type: 'index_parallel',
             spec: {
-              type: 'index',
-              ioConfig: { type: 'index', firehose: { type: 'inline', data: '{"test":"Data"}' } },
+              type: 'index_parallel',
+              ioConfig: {
+                type: 'index_parallel',
+                inputSource: { type: 'inline', data: '{"test":"Data"}' },
+                inputFormat: { type: 'json' },
+              },
               dataSchema: {
                 dataSource: 'sample',
-                parser: {
-                  type: 'string',
-                  parseSpec: {
-                    format: 'json',
-                    timestampSpec: {
-                      column: '!!!_no_such_column_!!!',
-                      missingValue: '2010-01-01T00:00:00Z',
-                    },
-                    dimensionsSpec: { dimensions: ['test'] },
-                  },
+                timestampSpec: {
+                  column: '!!!_no_such_column_!!!',
+                  missingValue: '2010-01-01T00:00:00Z',
                 },
+                dimensionsSpec: { dimensions: ['test'] },
                 transformSpec: {},
                 metricsSpec: [],
                 granularitySpec: { queryGranularity: 'NONE' },
diff --git a/web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap b/web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap
new file mode 100644
index 00000000000..031d6872dd5
--- /dev/null
+++ b/web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap
@@ -0,0 +1,76 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`ingestion-spec upgrades 1`] = `
+Object {
+  "dataSchema": Object {
+    "dataSource": "wikipedia",
+    "dimensionsSpec": Object {
+      "dimensions": Array [
+        "channel",
+        "cityName",
+        "comment",
+      ],
+    },
+    "granularitySpec": Object {
+      "queryGranularity": "HOUR",
+      "rollup": true,
+      "segmentGranularity": "DAY",
+      "type": "uniform",
+    },
+    "metricsSpec": Array [
+      Object {
+        "name": "count",
+        "type": "count",
+      },
+      Object {
+        "fieldName": "added",
+        "name": "sum_added",
+        "type": "longSum",
+      },
+    ],
+    "timestampSpec": Object {
+      "column": "timestamp",
+      "format": "iso",
+    },
+    "transformSpec": Object {
+      "filter": Object {
+        "dimension": "commentLength",
+        "type": "selector",
+        "value": "35",
+      },
+      "transforms": Array [
+        Object {
+          "expression": "concat(\\"channel\\", 'lol')",
+          "name": "channel",
+          "type": "expression",
+        },
+      ],
+    },
+  },
+  "ioConfig": Object {
+    "inputFormat": Object {
+      "flattenSpec": Object {
+        "fields": Array [
+          Object {
+            "expr": "$.cityName",
+            "name": "cityNameAlt",
+            "type": "path",
+          },
+        ],
+      },
+      "type": "json",
+    },
+    "inputSource": Object {
+      "type": "http",
+      "uris": Array [
+        "https://static.imply.io/data/wikipedia.json.gz",
+      ],
+    },
+    "type": "index_parallel",
+  },
+  "tuningConfig": Object {
+    "type": "index_parallel",
+  },
+  "type": "index_parallel",
+}
+`;
diff --git a/web-console/src/utils/druid-type.ts b/web-console/src/utils/druid-type.ts
index b0540d35cab..9f5a6f93980 100644
--- a/web-console/src/utils/druid-type.ts
+++ b/web-console/src/utils/druid-type.ts
@@ -88,17 +88,13 @@ export function updateSchemaWithSample(
   let newSpec = spec;
 
   if (dimensionMode === 'auto-detect') {
-    newSpec = deepSet(newSpec, 'dataSchema.parser.parseSpec.dimensionsSpec.dimensions', []);
+    newSpec = deepSet(newSpec, 'dataSchema.dimensionsSpec.dimensions', []);
   } else {
-    newSpec = deepDelete(newSpec, 'dataSchema.parser.parseSpec.dimensionsSpec.dimensionExclusions');
+    newSpec = deepDelete(newSpec, 'dataSchema.dimensionsSpec.dimensionExclusions');
 
     const dimensions = getDimensionSpecs(headerAndRows, rollup);
     if (dimensions) {
-      newSpec = deepSet(
-        newSpec,
-        'dataSchema.parser.parseSpec.dimensionsSpec.dimensions',
-        dimensions,
-      );
+      newSpec = deepSet(newSpec, 'dataSchema.dimensionsSpec.dimensions', dimensions);
     }
   }
 
diff --git a/web-console/src/utils/ingestion-spec.spec.ts b/web-console/src/utils/ingestion-spec.spec.ts
new file mode 100644
index 00000000000..f56617ccdf7
--- /dev/null
+++ b/web-console/src/utils/ingestion-spec.spec.ts
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { downgradeSpec, upgradeSpec } from './ingestion-spec';
+
+describe('ingestion-spec', () => {
+  const oldSpec = {
+    type: 'index_parallel',
+    ioConfig: {
+      type: 'index_parallel',
+      firehose: {
+        type: 'http',
+        uris: ['https://static.imply.io/data/wikipedia.json.gz'],
+      },
+    },
+    tuningConfig: {
+      type: 'index_parallel',
+    },
+    dataSchema: {
+      dataSource: 'wikipedia',
+      granularitySpec: {
+        type: 'uniform',
+        segmentGranularity: 'DAY',
+        queryGranularity: 'HOUR',
+        rollup: true,
+      },
+      parser: {
+        type: 'string',
+        parseSpec: {
+          format: 'json',
+          timestampSpec: {
+            column: 'timestamp',
+            format: 'iso',
+          },
+          dimensionsSpec: {
+            dimensions: ['channel', 'cityName', 'comment'],
+          },
+          flattenSpec: {
+            fields: [
+              {
+                type: 'path',
+                name: 'cityNameAlt',
+                expr: '$.cityName',
+              },
+            ],
+          },
+        },
+      },
+      transformSpec: {
+        transforms: [
+          {
+            type: 'expression',
+            name: 'channel',
+            expression: 'concat("channel", \'lol\')',
+          },
+        ],
+        filter: {
+          type: 'selector',
+          dimension: 'commentLength',
+          value: '35',
+        },
+      },
+      metricsSpec: [
+        {
+          name: 'count',
+          type: 'count',
+        },
+        {
+          name: 'sum_added',
+          type: 'longSum',
+          fieldName: 'added',
+        },
+      ],
+    },
+  };
+
+  it('upgrades', () => {
+    expect(upgradeSpec(oldSpec)).toMatchSnapshot();
+  });
+
+  it('round trips', () => {
+    expect(downgradeSpec(upgradeSpec(oldSpec))).toMatchObject(oldSpec);
+  });
+});
diff --git a/web-console/src/utils/ingestion-spec.tsx b/web-console/src/utils/ingestion-spec.tsx
index f67cba58e62..7908b565dc8 100644
--- a/web-console/src/utils/ingestion-spec.tsx
+++ b/web-console/src/utils/ingestion-spec.tsx
@@ -29,7 +29,7 @@ import {
   DATETIME_TIME_FORMATS,
   OTHER_TIME_FORMATS,
 } from './druid-time';
-import { deepGet, deepSet } from './object-change';
+import { deepDelete, deepGet, deepMove, deepSet } from './object-change';
 
 export const MAX_INLINE_DATA_LENGTH = 65536;
 
@@ -41,8 +41,8 @@ const CURRENT_YEAR = new Date().getUTCFullYear();
 
 export interface IngestionSpec {
   type?: IngestionType;
-  dataSchema: DataSchema;
   ioConfig: IoConfig;
+  dataSchema: DataSchema;
   tuningConfig?: TuningConfig;
 }
 
@@ -50,34 +50,37 @@ export function isEmptyIngestionSpec(spec: IngestionSpec) {
   return Object.keys(spec).length === 0;
 }
 
-export type IngestionType = 'kafka' | 'kinesis' | 'index_hadoop' | 'index' | 'index_parallel';
+export type IngestionType = 'kafka' | 'kinesis' | 'index_parallel';
 
-// A combination of IngestionType and firehose
+// A combination of IngestionType and inputSourceType
 export type IngestionComboType =
   | 'kafka'
   | 'kinesis'
-  | 'index:http'
-  | 'index:local'
-  | 'index:ingestSegment'
-  | 'index:inline'
-  | 'index:static-s3'
-  | 'index:static-google-blobstore'
-  | 'index:hdfs';
+  | 'index_parallel:http'
+  | 'index_parallel:local'
+  | 'index_parallel:druid'
+  | 'index_parallel:inline'
+  | 'index_parallel:s3'
+  | 'index_parallel:google'
+  | 'index_parallel:hdfs';
 
 // Some extra values that can be selected in the initial screen
 export type IngestionComboTypeWithExtra = IngestionComboType | 'hadoop' | 'example' | 'other';
 
+export function adjustIngestionSpec(spec: IngestionSpec) {
+  if (spec.tuningConfig) {
+    spec = deepSet(spec, 'tuningConfig', adjustTuningConfig(spec.tuningConfig));
+  }
+  return spec;
+}
+
 function ingestionTypeToIoAndTuningConfigType(ingestionType: IngestionType): string {
   switch (ingestionType) {
     case 'kafka':
     case 'kinesis':
-    case 'index':
     case 'index_parallel':
       return ingestionType;
 
-    case 'index_hadoop':
-      return 'hadoop';
-
     default:
       throw new Error(`unknown type '${ingestionType}'`);
   }
@@ -91,18 +94,17 @@ export function getIngestionComboType(spec: IngestionSpec): IngestionComboType |
     case 'kinesis':
       return ioConfig.type;
 
-    case 'index':
     case 'index_parallel':
-      const firehose = deepGet(spec, 'ioConfig.firehose') || EMPTY_OBJECT;
-      switch (firehose.type) {
+      const inputSource = deepGet(spec, 'ioConfig.inputSource') || EMPTY_OBJECT;
+      switch (inputSource.type) {
         case 'local':
         case 'http':
-        case 'ingestSegment':
+        case 'druid':
         case 'inline':
-        case 'static-s3':
-        case 'static-google-blobstore':
+        case 's3':
+        case 'google':
         case 'hdfs':
-          return `index:${firehose.type}` as IngestionComboType;
+          return `${ioConfig.type}:${inputSource.type}` as IngestionComboType;
       }
   }
 
@@ -111,25 +113,25 @@ export function getIngestionComboType(spec: IngestionSpec): IngestionComboType |
 
 export function getIngestionTitle(ingestionType: IngestionComboTypeWithExtra): string {
   switch (ingestionType) {
-    case 'index:local':
+    case 'index_parallel:local':
       return 'Local disk';
 
-    case 'index:http':
+    case 'index_parallel:http':
       return 'HTTP(s)';
 
-    case 'index:ingestSegment':
+    case 'index_parallel:druid':
       return 'Reindex from Druid';
 
-    case 'index:inline':
+    case 'index_parallel:inline':
       return 'Paste data';
 
-    case 'index:static-s3':
+    case 'index_parallel:s3':
       return 'Amazon S3';
 
-    case 'index:static-google-blobstore':
+    case 'index_parallel:google':
       return 'Google Cloud Storage';
 
-    case 'index:hdfs':
+    case 'index_parallel:hdfs':
       return 'HDFS';
 
     case 'kafka':
@@ -175,13 +177,13 @@ export function getIngestionDocLink(spec: IngestionSpec): string {
 
 export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra): string | undefined {
   switch (ingestionType) {
-    case 'index:static-s3':
+    case 'index_parallel:s3':
       return 'druid-s3-extensions';
 
-    case 'index:static-google-blobstore':
+    case 'index_parallel:google':
       return 'druid-google-extensions';
 
-    case 'index:hdfs':
+    case 'index_parallel:hdfs':
       return 'druid-hdfs-storage';
 
     case 'kafka':
@@ -199,46 +201,28 @@ export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra): s
 
 export interface DataSchema {
   dataSource: string;
-  parser: Parser;
+  timestampSpec: TimestampSpec;
   transformSpec?: TransformSpec;
   granularitySpec?: GranularitySpec;
+  dimensionsSpec: DimensionsSpec;
   metricsSpec?: MetricSpec[];
 }
 
-export interface Parser {
-  type?: string;
-  parseSpec: ParseSpec;
-}
-
-export interface ParseSpec {
-  format: string;
-  hasHeaderRow?: boolean;
+export interface InputFormat {
+  type: string;
+  findColumnsFromHeader?: boolean;
   skipHeaderRows?: number;
   columns?: string[];
   listDelimiter?: string;
   pattern?: string;
   function?: string;
-
-  timestampSpec: TimestampSpec;
-  dimensionsSpec: DimensionsSpec;
   flattenSpec?: FlattenSpec;
 }
 
-export function hasParallelAbility(spec: IngestionSpec): boolean {
-  const specType = getSpecType(spec);
-  return specType === 'index' || specType === 'index_parallel';
-}
-
-export function isParallel(spec: IngestionSpec): boolean {
-  const specType = getSpecType(spec);
-  return specType === 'index_parallel';
-}
-
 export type DimensionMode = 'specific' | 'auto-detect';
 
 export function getDimensionMode(spec: IngestionSpec): DimensionMode {
-  const dimensions =
-    deepGet(spec, 'dataSchema.parser.parseSpec.dimensionsSpec.dimensions') || EMPTY_ARRAY;
+  const dimensions = deepGet(spec, 'dataSchema.dimensionsSpec.dimensions') || EMPTY_ARRAY;
   return Array.isArray(dimensions) && dimensions.length === 0 ? 'auto-detect' : 'specific';
 }
 
@@ -247,9 +231,12 @@ export function getRollup(spec: IngestionSpec): boolean {
   return typeof specRollup === 'boolean' ? specRollup : true;
 }
 
-export function getSpecType(spec: Partial<IngestionSpec>): IngestionType | undefined {
+export function getSpecType(spec: Partial<IngestionSpec>): IngestionType {
   return (
-    deepGet(spec, 'type') || deepGet(spec, 'ioConfig.type') || deepGet(spec, 'tuningConfig.type')
+    deepGet(spec, 'type') ||
+    deepGet(spec, 'ioConfig.type') ||
+    deepGet(spec, 'tuningConfig.type') ||
+    'index_parallel'
   );
 }
 
@@ -261,18 +248,8 @@ export function isTask(spec: IngestionSpec) {
   );
 }
 
-export function isIngestSegment(spec: IngestionSpec): boolean {
-  return deepGet(spec, 'ioConfig.firehose.type') === 'ingestSegment';
-}
-
-export function changeParallel(spec: IngestionSpec, parallel: boolean): IngestionSpec {
-  if (!hasParallelAbility(spec)) return spec;
-  const newType = parallel ? 'index_parallel' : 'index';
-  let newSpec = spec;
-  newSpec = deepSet(newSpec, 'type', newType);
-  newSpec = deepSet(newSpec, 'ioConfig.type', newType);
-  newSpec = deepSet(newSpec, 'tuningConfig.type', newType);
-  return newSpec;
+export function isDruidSource(spec: IngestionSpec): boolean {
+  return deepGet(spec, 'ioConfig.inputSource.type') === 'druid';
 }
 
 /**
@@ -296,12 +273,12 @@ export function normalizeSpec(spec: Partial<IngestionSpec>): IngestionSpec {
   return spec as IngestionSpec;
 }
 
-const PARSE_SPEC_FORM_FIELDS: Field<ParseSpec>[] = [
+const INPUT_FORMAT_FORM_FIELDS: Field<InputFormat>[] = [
   {
-    name: 'format',
-    label: 'Parser to use',
+    name: 'type',
+    label: 'Input format',
     type: 'string',
-    suggestions: ['json', 'csv', 'tsv', 'regex'],
+    suggestions: ['json', 'csv', 'tsv', 'regex', 'parquet', 'orc'],
     info: (
       <>
         <p>The parser used to parse the data.</p>
@@ -321,25 +298,25 @@ const PARSE_SPEC_FORM_FIELDS: Field<ParseSpec>[] = [
     name: 'pattern',
     type: 'string',
     required: true,
-    defined: (p: ParseSpec) => p.format === 'regex',
+    defined: (p: InputFormat) => p.type === 'regex',
   },
   {
     name: 'function',
     type: 'string',
     required: true,
-    defined: (p: ParseSpec) => p.format === 'javascript',
+    defined: (p: InputFormat) => p.type === 'javascript',
   },
   {
-    name: 'hasHeaderRow',
+    name: 'findColumnsFromHeader',
     type: 'boolean',
-    defaultValue: false,
-    defined: (p: ParseSpec) => p.format === 'csv' || p.format === 'tsv',
+    required: true,
+    defined: (p: InputFormat) => p.type === 'csv' || p.type === 'tsv',
   },
   {
     name: 'skipHeaderRows',
     type: 'number',
     defaultValue: 0,
-    defined: (p: ParseSpec) => p.format === 'csv' || p.format === 'tsv',
+    defined: (p: InputFormat) => p.type === 'csv' || p.type === 'tsv',
     min: 0,
     info: (
       <>
@@ -352,51 +329,60 @@ const PARSE_SPEC_FORM_FIELDS: Field<ParseSpec>[] = [
   {
     name: 'columns',
     type: 'string-array',
-    required: (p: ParseSpec) =>
-      ((p.format === 'csv' || p.format === 'tsv') && !p.hasHeaderRow) || p.format === 'regex',
-    defined: (p: ParseSpec) =>
-      ((p.format === 'csv' || p.format === 'tsv') && !p.hasHeaderRow) || p.format === 'regex',
-  },
-  {
-    name: 'delimiter',
-    type: 'string',
-    defaultValue: '\t',
-    defined: (p: ParseSpec) => p.format === 'tsv',
-    info: <>A custom delimiter for data values.</>,
+    required: (p: InputFormat) =>
+      ((p.type === 'csv' || p.type === 'tsv') && !p.findColumnsFromHeader) || p.type === 'regex',
+    defined: (p: InputFormat) =>
+      ((p.type === 'csv' || p.type === 'tsv') && !p.findColumnsFromHeader) || p.type === 'regex',
   },
+  // {
+  //   name: 'delimiter',
+  //   type: 'string',
+  //   defaultValue: '\t',
+  //   defined: (p: InputFormat) => p.type === 'tsv',
+  //   info: <>A custom delimiter for data values.</>,
+  // },
   {
     name: 'listDelimiter',
     type: 'string',
-    defined: (p: ParseSpec) => p.format === 'csv' || p.format === 'tsv',
+    defined: (p: InputFormat) => p.type === 'csv' || p.type === 'tsv' || p.type === 'regex',
     info: <>A custom delimiter for multi-value dimensions.</>,
   },
+  {
+    name: 'binaryAsString',
+    type: 'boolean',
+    defaultValue: false,
+    defined: (p: InputFormat) => p.type === 'parquet' || p.type === 'orc',
+    info: (
+      <>
+        Specifies if the bytes parquet column which is not logically marked as a string or enum type
+        should be treated as a UTF-8 encoded string.
+      </>
+    ),
+  },
 ];
 
-export function getParseSpecFormFields() {
-  return PARSE_SPEC_FORM_FIELDS;
+export function getInputFormatFormFields() {
+  return INPUT_FORMAT_FORM_FIELDS;
 }
 
-export function issueWithParser(parser: Parser | undefined): string | undefined {
-  if (!parser) return 'no parser';
-  if (parser.type === 'map') return;
-
-  const { parseSpec } = parser;
-  if (!parseSpec) return 'no parse spec';
-  if (!parseSpec.format) return 'missing a format';
-  switch (parseSpec.format) {
+export function issueWithInputFormat(inputFormat: InputFormat | undefined): string | undefined {
+  if (!inputFormat) return 'no input format';
+  if (!inputFormat.type) return 'missing a type';
+  switch (inputFormat.type) {
     case 'regex':
-      if (!parseSpec.pattern) return "must have a 'pattern'";
+      if (!inputFormat.pattern) return "must have a 'pattern'";
       break;
 
     case 'javascript':
-      if (!parseSpec['function']) return "must have a 'function'";
+      if (!inputFormat['function']) return "must have a 'function'";
       break;
   }
   return;
 }
 
-export function parseSpecHasFlatten(parseSpec: ParseSpec): boolean {
-  return parseSpec.format === 'json';
+export function inputFormatCanFlatten(inputFormat: InputFormat): boolean {
+  const inputFormatType = inputFormat.type;
+  return inputFormatType === 'json' || inputFormatType === 'parquet' || inputFormatType === 'orc';
 }
 
 export interface TimestampSpec {
@@ -412,13 +398,22 @@ export function getTimestampSpecColumn(timestampSpec: TimestampSpec) {
 
 const NO_SUCH_COLUMN = '!!!_no_such_column_!!!';
 
-const EMPTY_TIMESTAMP_SPEC: TimestampSpec = {
+const DUMMY_TIMESTAMP_SPEC: TimestampSpec = {
+  column: NO_SUCH_COLUMN,
+  missingValue: '1970-01-01T00:00:00Z',
+};
+
+export function getDummyTimestampSpec() {
+  return DUMMY_TIMESTAMP_SPEC;
+}
+
+const CONSTANT_TIMESTAMP_SPEC: TimestampSpec = {
   column: NO_SUCH_COLUMN,
   missingValue: '2010-01-01T00:00:00Z',
 };
 
-export function getEmptyTimestampSpec() {
-  return EMPTY_TIMESTAMP_SPEC;
+export function getConstantTimestampSpec() {
+  return CONSTANT_TIMESTAMP_SPEC;
 }
 
 export function isColumnTimestampSpec(timestampSpec: TimestampSpec) {
@@ -936,7 +931,8 @@ export function getMetricSpecName(metricSpec: MetricSpec): string {
 
 export interface IoConfig {
   type: string;
-  firehose?: Firehose;
+  inputSource?: InputSource;
+  inputFormat?: InputFormat;
   appendToExisting?: boolean;
   topic?: string;
   consumerProperties?: any;
@@ -958,7 +954,7 @@ export function invalidIoConfig(ioConfig: IoConfig): boolean {
   );
 }
 
-export interface Firehose {
+export interface InputSource {
   type: string;
   baseDir?: string;
   filter?: any;
@@ -967,7 +963,7 @@ export interface Firehose {
   blobs?: { bucket: string; path: string }[];
   fetchTimeout?: number;
 
-  // ingestSegment
+  // druid
   dataSource?: string;
   interval?: string;
   dimensions?: string[];
@@ -982,30 +978,30 @@ export interface Firehose {
 }
 
 export function getIoConfigFormFields(ingestionComboType: IngestionComboType): Field<IoConfig>[] {
-  const firehoseType: Field<IoConfig> = {
-    name: 'firehose.type',
-    label: 'Firehose type',
+  const inputSourceType: Field<IoConfig> = {
+    name: 'inputSource.type',
+    label: 'Source type',
     type: 'string',
-    suggestions: ['local', 'http', 'inline', 'static-s3', 'static-google-blobstore', 'hdfs'],
+    suggestions: ['local', 'http', 'inline', 's3', 'google', 'hdfs'],
     info: (
       <p>
         Druid connects to raw data through{' '}
         <ExternalLink
           href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/ingestion/firehose.html`}
         >
-          firehoses
+          inputSources
         </ExternalLink>
-        . You can change your selected firehose here.
+        . You can change your selected inputSource here.
       </p>
     ),
   };
 
   switch (ingestionComboType) {
-    case 'index:http':
+    case 'index_parallel:http':
       return [
-        firehoseType,
+        inputSourceType,
         {
-          name: 'firehose.uris',
+          name: 'inputSource.uris',
           label: 'URIs',
           type: 'string-array',
           placeholder:
@@ -1019,14 +1015,14 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           ),
         },
         {
-          name: 'firehose.httpAuthenticationUsername',
+          name: 'inputSource.httpAuthenticationUsername',
           label: 'HTTP auth username',
           type: 'string',
           placeholder: '(optional)',
           info: <p>Username to use for authentication with specified URIs</p>,
         },
         {
-          name: 'firehose.httpAuthenticationPassword',
+          name: 'inputSource.httpAuthenticationPassword',
           label: 'HTTP auth password',
           type: 'string',
           placeholder: '(optional)',
@@ -1034,11 +1030,11 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
         },
       ];
 
-    case 'index:local':
+    case 'index_parallel:local':
       return [
-        firehoseType,
+        inputSourceType,
         {
-          name: 'firehose.baseDir',
+          name: 'inputSource.baseDir',
           label: 'Base directory',
           type: 'string',
           placeholder: '/path/to/files/',
@@ -1048,24 +1044,24 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
               <ExternalLink
                 href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/ingestion/firehose.html#localfirehose`}
               >
-                firehose.baseDir
+                inputSource.baseDir
               </ExternalLink>
               <p>Specifies the directory to search recursively for files to be ingested.</p>
             </>
           ),
         },
         {
-          name: 'firehose.filter',
+          name: 'inputSource.filter',
           label: 'File filter',
           type: 'string',
           required: true,
-          suggestions: ['*', '*.json', '*.json.gz', '*.csv', '*.tsv'],
+          suggestions: ['*', '*.json', '*.json.gz', '*.csv', '*.tsv', '*.parquet', '*.orc'],
           info: (
             <>
               <ExternalLink
                 href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/ingestion/firehose.html#localfirehose`}
               >
-                firehose.filter
+                inputSource.filter
               </ExternalLink>
               <p>
                 A wildcard filter for files. See{' '}
@@ -1079,18 +1075,18 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
         },
       ];
 
-    case 'index:ingestSegment':
+    case 'index_parallel:druid':
       return [
-        firehoseType,
+        inputSourceType,
         {
-          name: 'firehose.dataSource',
+          name: 'inputSource.dataSource',
           label: 'Datasource',
           type: 'string',
           required: true,
           info: <p>The datasource to fetch rows from.</p>,
         },
         {
-          name: 'firehose.interval',
+          name: 'inputSource.interval',
           label: 'Interval',
           type: 'interval',
           placeholder: `${CURRENT_YEAR}-01-01/${CURRENT_YEAR + 1}-01-01`,
@@ -1103,7 +1099,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           ),
         },
         {
-          name: 'firehose.dimensions',
+          name: 'inputSource.dimensions',
           label: 'Dimensions',
           type: 'string-array',
           placeholder: '(optional)',
@@ -1115,7 +1111,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           ),
         },
         {
-          name: 'firehose.metrics',
+          name: 'inputSource.metrics',
           label: 'Metrics',
           type: 'string-array',
           placeholder: '(optional)',
@@ -1127,7 +1123,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           ),
         },
         {
-          name: 'firehose.filter',
+          name: 'inputSource.filter',
           label: 'Filter',
           type: 'json',
           placeholder: '(optional)',
@@ -1145,22 +1141,22 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
         },
       ];
 
-    case 'index:inline':
+    case 'index_parallel:inline':
       return [
-        firehoseType,
+        inputSourceType,
         // do not add 'data' here as it has special handling in the load-data view
       ];
 
-    case 'index:static-s3':
+    case 'index_parallel:s3':
       return [
-        firehoseType,
+        inputSourceType,
         {
-          name: 'firehose.uris',
+          name: 'inputSource.uris',
           label: 'S3 URIs',
           type: 'string-array',
           placeholder: 's3://your-bucket/some-file1.ext, s3://your-bucket/some-file2.ext',
           required: true,
-          defined: ioConfig => !deepGet(ioConfig, 'firehose.prefixes'),
+          defined: ioConfig => !deepGet(ioConfig, 'inputSource.prefixes'),
           info: (
             <>
               <p>
@@ -1172,12 +1168,12 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           ),
         },
         {
-          name: 'firehose.prefixes',
+          name: 'inputSource.prefixes',
           label: 'S3 prefixes',
           type: 'string-array',
           placeholder: 's3://your-bucket/some-path1, s3://your-bucket/some-path2',
           required: true,
-          defined: ioConfig => !deepGet(ioConfig, 'firehose.uris'),
+          defined: ioConfig => !deepGet(ioConfig, 'inputSource.uris'),
           info: (
             <>
               <p>A list of paths (with bucket) where your files are stored.</p>
@@ -1187,11 +1183,11 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
         },
       ];
 
-    case 'index:static-google-blobstore':
+    case 'index_parallel:google':
       return [
-        firehoseType,
+        inputSourceType,
         {
-          name: 'firehose.blobs',
+          name: 'inputSource.blobs',
           label: 'Google blobs',
           type: 'json',
           required: true,
@@ -1211,11 +1207,11 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
         },
       ];
 
-    case 'index:hdfs':
+    case 'index_parallel:hdfs':
       return [
-        firehoseType,
+        inputSourceType,
         {
-          name: 'firehose.paths',
+          name: 'inputSource.paths',
           label: 'Paths',
           type: 'string',
           placeholder: '/path/to/file.ext',
@@ -1341,44 +1337,44 @@ function nonEmptyArray(a: any) {
   return Array.isArray(a) && Boolean(a.length);
 }
 
-function issueWithFirehose(firehose: Firehose | undefined): string | undefined {
-  if (!firehose) return 'does not exist';
-  if (!firehose.type) return 'missing a type';
-  switch (firehose.type) {
+function issueWithInputSource(inputSource: InputSource | undefined): string | undefined {
+  if (!inputSource) return 'does not exist';
+  if (!inputSource.type) return 'missing a type';
+  switch (inputSource.type) {
     case 'local':
-      if (!firehose.baseDir) return `must have a 'baseDir'`;
-      if (!firehose.filter) return `must have a 'filter'`;
+      if (!inputSource.baseDir) return `must have a 'baseDir'`;
+      if (!inputSource.filter) return `must have a 'filter'`;
       break;
 
     case 'http':
-      if (!nonEmptyArray(firehose.uris)) {
+      if (!nonEmptyArray(inputSource.uris)) {
         return 'must have at least one uri';
       }
       break;
 
-    case 'ingestSegment':
-      if (!firehose.dataSource) return `must have a 'dataSource'`;
-      if (!firehose.interval) return `must have an 'interval'`;
+    case 'druid':
+      if (!inputSource.dataSource) return `must have a 'dataSource'`;
+      if (!inputSource.interval) return `must have an 'interval'`;
       break;
 
     case 'inline':
-      if (!firehose.data) return `must have 'data'`;
+      if (!inputSource.data) return `must have 'data'`;
       break;
 
-    case 'static-s3':
-      if (!nonEmptyArray(firehose.uris) && !nonEmptyArray(firehose.prefixes)) {
+    case 's3':
+      if (!nonEmptyArray(inputSource.uris) && !nonEmptyArray(inputSource.prefixes)) {
         return 'must have at least one uri or prefix';
       }
       break;
 
-    case 'static-google-blobstore':
-      if (!nonEmptyArray(firehose.blobs)) {
+    case 'google':
+      if (!nonEmptyArray(inputSource.blobs)) {
         return 'must have at least one blob';
       }
       break;
 
     case 'hdfs':
-      if (!firehose.paths) {
+      if (!inputSource.paths) {
         return 'must have paths';
       }
       break;
@@ -1386,14 +1382,17 @@ function issueWithFirehose(firehose: Firehose | undefined): string | undefined {
   return;
 }
 
-export function issueWithIoConfig(ioConfig: IoConfig | undefined): string | undefined {
+export function issueWithIoConfig(
+  ioConfig: IoConfig | undefined,
+  ignoreInputFormat = false,
+): string | undefined {
   if (!ioConfig) return 'does not exist';
   if (!ioConfig.type) return 'missing a type';
   switch (ioConfig.type) {
     case 'index':
     case 'index_parallel':
-      if (issueWithFirehose(ioConfig.firehose)) {
-        return `firehose: '${issueWithFirehose(ioConfig.firehose)}'`;
+      if (issueWithInputSource(ioConfig.inputSource)) {
+        return `inputSource: '${issueWithInputSource(ioConfig.inputSource)}'`;
       }
       break;
 
@@ -1406,6 +1405,10 @@ export function issueWithIoConfig(ioConfig: IoConfig | undefined): string | unde
       break;
   }
 
+  if (!ignoreInputFormat && issueWithInputFormat(ioConfig.inputFormat)) {
+    return `inputFormat: '${issueWithInputFormat(ioConfig.inputFormat)}'`;
+  }
+
   return;
 }
 
@@ -1413,13 +1416,13 @@ export function getIoConfigTuningFormFields(
   ingestionComboType: IngestionComboType,
 ): Field<IoConfig>[] {
   switch (ingestionComboType) {
-    case 'index:http':
-    case 'index:static-s3':
-    case 'index:static-google-blobstore':
-    case 'index:hdfs':
+    case 'index_parallel:http':
+    case 'index_parallel:s3':
+    case 'index_parallel:google':
+    case 'index_parallel:hdfs':
       return [
         {
-          name: 'firehose.fetchTimeout',
+          name: 'inputSource.fetchTimeout',
           label: 'Fetch timeout',
           type: 'number',
           defaultValue: 60000,
@@ -1430,7 +1433,7 @@ export function getIoConfigTuningFormFields(
           ),
         },
         {
-          name: 'firehose.maxFetchRetry',
+          name: 'inputSource.maxFetchRetry',
           label: 'Max fetch retry',
           type: 'number',
           defaultValue: 3,
@@ -1441,7 +1444,7 @@ export function getIoConfigTuningFormFields(
           ),
         },
         {
-          name: 'firehose.maxCacheCapacityBytes',
+          name: 'inputSource.maxCacheCapacityBytes',
           label: 'Max cache capacity bytes',
           type: 'number',
           defaultValue: 1073741824,
@@ -1455,7 +1458,7 @@ export function getIoConfigTuningFormFields(
           ),
         },
         {
-          name: 'firehose.maxFetchCapacityBytes',
+          name: 'inputSource.maxFetchCapacityBytes',
           label: 'Max fetch capacity bytes',
           type: 'number',
           defaultValue: 1073741824,
@@ -1469,7 +1472,7 @@ export function getIoConfigTuningFormFields(
           ),
         },
         {
-          name: 'firehose.prefetchTriggerBytes',
+          name: 'inputSource.prefetchTriggerBytes',
           label: 'Prefetch trigger bytes',
           type: 'number',
           placeholder: 'maxFetchCapacityBytes / 2',
@@ -1481,14 +1484,14 @@ export function getIoConfigTuningFormFields(
         },
       ];
 
-    case 'index:local':
-    case 'index:inline':
+    case 'index_parallel:local':
+    case 'index_parallel:inline':
       return [];
 
-    case 'index:ingestSegment':
+    case 'index_parallel:druid':
       return [
         {
-          name: 'firehose.maxFetchCapacityBytes',
+          name: 'inputSource.maxFetchCapacityBytes',
           label: 'Max fetch capacity bytes',
           type: 'number',
           defaultValue: 157286400,
@@ -1738,28 +1741,31 @@ export function guessDataSourceName(spec: IngestionSpec): string | undefined {
   switch (ioConfig.type) {
     case 'index':
     case 'index_parallel':
-      const firehose = ioConfig.firehose;
-      if (!firehose) return;
+      const inputSource = ioConfig.inputSource;
+      if (!inputSource) return;
 
-      switch (firehose.type) {
+      switch (inputSource.type) {
         case 'local':
-          if (firehose.filter && filterIsFilename(firehose.filter)) {
-            return basenameFromFilename(firehose.filter);
-          } else if (firehose.baseDir) {
-            return filenameFromPath(firehose.baseDir);
+          if (inputSource.filter && filterIsFilename(inputSource.filter)) {
+            return basenameFromFilename(inputSource.filter);
+          } else if (inputSource.baseDir) {
+            return filenameFromPath(inputSource.baseDir);
           } else {
             return;
           }
 
-        case 'static-s3':
-          const s3Path = (firehose.uris || EMPTY_ARRAY)[0] || (firehose.prefixes || EMPTY_ARRAY)[0];
+        case 's3':
+          const s3Path =
+            (inputSource.uris || EMPTY_ARRAY)[0] || (inputSource.prefixes || EMPTY_ARRAY)[0];
           return s3Path ? filenameFromPath(s3Path) : undefined;
 
         case 'http':
-          return Array.isArray(firehose.uris) ? filenameFromPath(firehose.uris[0]) : undefined;
+          return Array.isArray(inputSource.uris)
+            ? filenameFromPath(inputSource.uris[0])
+            : undefined;
 
-        case 'ingestSegment':
-          return firehose.dataSource;
+        case 'druid':
+          return inputSource.dataSource;
 
         case 'inline':
           return 'inline_data';
@@ -1784,8 +1790,7 @@ export interface TuningConfig {
   type: string;
   maxRowsInMemory?: number;
   maxBytesInMemory?: number;
-  maxTotalRows?: number;
-  numShards?: number;
+  partitionsSpec?: PartitionsSpec;
   maxPendingPersists?: number;
   indexSpec?: IndexSpec;
   forceExtendableShardSpecs?: boolean;
@@ -1811,19 +1816,66 @@ export interface TuningConfig {
   fetchThreads?: number;
 }
 
+export interface PartitionsSpec {
+  type: 'string';
+
+  // For type: dynamic
+  maxTotalRows?: number;
+
+  // For type: hashed
+  numShards?: number;
+  partitionDimensions?: string[];
+
+  // For type: single_dim
+  targetRowsPerSegment?: number;
+  maxRowsPerSegment?: number;
+  partitionDimension?: string;
+  assumeGrouped?: boolean;
+}
+
+export function adjustTuningConfig(tuningConfig: TuningConfig) {
+  const tuningConfigType = deepGet(tuningConfig, 'type');
+  if (tuningConfigType !== 'index_parallel') return tuningConfig;
+
+  const partitionsSpecType = deepGet(tuningConfig, 'partitionsSpec.type');
+  if (tuningConfig.forceGuaranteedRollup) {
+    if (partitionsSpecType !== 'hashed' && partitionsSpecType !== 'single_dim') {
+      tuningConfig = deepSet(tuningConfig, 'partitionsSpec', { type: 'hashed' });
+    }
+  } else {
+    if (partitionsSpecType !== 'dynamic') {
+      tuningConfig = deepSet(tuningConfig, 'partitionsSpec', { type: 'dynamic' });
+    }
+  }
+  return tuningConfig;
+}
+
 export function invalidTuningConfig(tuningConfig: TuningConfig, intervals: any): boolean {
-  return Boolean(
-    tuningConfig.type === 'index_parallel' &&
-      tuningConfig.forceGuaranteedRollup &&
-      (!tuningConfig.numShards || !intervals),
-  );
+  if (tuningConfig.type !== 'index_parallel' || !tuningConfig.forceGuaranteedRollup) return false;
+
+  if (!intervals) return true;
+  switch (deepGet(tuningConfig, 'partitionsSpec.type')) {
+    case 'hashed':
+      if (!deepGet(tuningConfig, 'partitionsSpec.numShards')) return true;
+      break;
+
+    case 'single_dim':
+      if (!deepGet(tuningConfig, 'partitionsSpec.partitionDimension')) return true;
+      if (
+        !deepGet(tuningConfig, 'partitionsSpec.targetRowsPerSegment') &&
+        !deepGet(tuningConfig, 'partitionsSpec.maxRowsPerSegment')
+      ) {
+        return true;
+      }
+  }
+
+  return false;
 }
 
 export function getPartitionRelatedTuningSpecFormFields(
   specType: IngestionType,
 ): Field<TuningConfig>[] {
   switch (specType) {
-    case 'index':
     case 'index_parallel':
       return [
         {
@@ -1831,23 +1883,53 @@ export function getPartitionRelatedTuningSpecFormFields(
           type: 'boolean',
           defaultValue: false,
           info: (
-            <>
-              <p>
-                Forces guaranteeing the perfect rollup. The perfect rollup optimizes the total size
-                of generated segments and querying time while indexing time will be increased. If
-                this is set to true, the index task will read the entire input data twice: one for
-                finding the optimal number of partitions per time chunk and one for generating
-                segments.
-              </p>
-            </>
+            <p>
+              Forces guaranteeing the perfect rollup. The perfect rollup optimizes the total size of
+              generated segments and querying time while indexing time will be increased. If this is
+              set to true, the index task will read the entire input data twice: one for finding the
+              optimal number of partitions per time chunk and one for generating segments.
+            </p>
           ),
+          adjustment: adjustTuningConfig,
         },
         {
-          name: 'numShards', // This is mandatory if index_parallel and forceGuaranteedRollup
+          name: 'partitionsSpec.type',
+          label: 'Partitioning type',
+          type: 'string',
+          suggestions: (t: TuningConfig) =>
+            t.forceGuaranteedRollup ? ['hashed', 'single_dim'] : ['dynamic'],
+          info: (
+            <p>
+              For perfect rollup, you should use either <Code>hashed</Code> (partitioning based on
+              the hash of dimensions in each row) or <Code>single_dim</Code> (based on ranges of a
+              single dimension. For best-effort rollup, you should use dynamic.
+            </p>
+          ),
+        },
+        // partitionsSpec type: dynamic
+        {
+          name: 'partitionsSpec.maxRowsPerSegment',
+          label: 'Max rows per segment',
           type: 'number',
-          defined: (t: TuningConfig) => Boolean(t.forceGuaranteedRollup),
-          required: (t: TuningConfig) =>
-            Boolean(t.type === 'index_parallel' && t.forceGuaranteedRollup),
+          defaultValue: 5000000,
+          defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'dynamic',
+          info: <>Determines how many rows are in each segment.</>,
+        },
+        {
+          name: 'partitionsSpec.maxTotalRows',
+          label: 'Max total rows',
+          type: 'number',
+          defaultValue: 20000000,
+          defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'dynamic',
+          info: <>Total number of rows in segments waiting for being pushed.</>,
+        },
+        // partitionsSpec type: hashed
+        {
+          name: 'partitionsSpec.numShards',
+          label: 'Num shards',
+          type: 'number',
+          defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'hashed',
+          required: true,
           info: (
             <>
               Directly specify the number of shards to create. If this is specified and 'intervals'
@@ -1858,32 +1940,60 @@ export function getPartitionRelatedTuningSpecFormFields(
           ),
         },
         {
-          name: 'partitionDimensions',
+          name: 'partitionsSpec.partitionDimensions',
+          label: 'Partition dimensions',
           type: 'string-array',
-          defined: (t: TuningConfig) => Boolean(t.forceGuaranteedRollup),
+          defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'hashed',
+          info: <p>The dimensions to partition on. Leave blank to select all dimensions.</p>,
+        },
+        // partitionsSpec type: single_dim
+        {
+          name: 'partitionsSpec.partitionDimension',
+          label: 'Partition dimension',
+          type: 'string',
+          defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'single_dim',
+          required: true,
+          info: <p>The dimension to partition on.</p>,
+        },
+        {
+          name: 'partitionsSpec.targetRowsPerSegment',
+          label: 'Target rows per segment',
+          type: 'number',
+          zeroMeansUndefined: true,
+          defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'single_dim',
+          required: (t: TuningConfig) =>
+            !deepGet(t, 'partitionsSpec.targetRowsPerSegment') &&
+            !deepGet(t, 'partitionsSpec.maxRowsPerSegment'),
           info: (
-            <>
-              <p>Does not currently work with parallel ingestion</p>
-              <p>
-                The dimensions to partition on. Leave blank to select all dimensions. Only used with
-                forceGuaranteedRollup = true, will be ignored otherwise.
-              </p>
-            </>
+            <p>
+              Target number of rows to include in a partition, should be a number that targets
+              segments of 500MB~1GB.
+            </p>
           ),
         },
         {
-          name: 'maxRowsPerSegment',
+          name: 'partitionsSpec.maxRowsPerSegment',
+          label: 'Max rows per segment',
           type: 'number',
-          defaultValue: 5000000,
-          defined: (t: TuningConfig) => !t.forceGuaranteedRollup && t.numShards == null, // Can not be set if numShards is specified
-          info: <>Determines how many rows are in each segment.</>,
+          zeroMeansUndefined: true,
+          defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'single_dim',
+          required: (t: TuningConfig) =>
+            !deepGet(t, 'partitionsSpec.targetRowsPerSegment') &&
+            !deepGet(t, 'partitionsSpec.maxRowsPerSegment'),
+          info: <p>Maximum number of rows to include in a partition.</p>,
         },
         {
-          name: 'maxTotalRows',
-          type: 'number',
-          defaultValue: 20000000,
-          defined: (t: TuningConfig) => !t.forceGuaranteedRollup,
-          info: <>Total number of rows in segments waiting for being pushed.</>,
+          name: 'partitionsSpec.assumeGrouped',
+          label: 'Assume grouped',
+          type: 'boolean',
+          defaultValue: false,
+          defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'single_dim',
+          info: (
+            <p>
+              Assume that input data has already been grouped on time and dimensions. Ingestion will
+              run faster, but may choose sub-optimal partitions if this assumption is violated.
+            </p>
+          ),
         },
       ];
 
@@ -1913,6 +2023,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     name: 'maxNumConcurrentSubTasks',
     type: 'number',
     defaultValue: 1,
+    min: 1,
     defined: (t: TuningConfig) => t.type === 'index_parallel',
     info: (
       <>
@@ -2237,8 +2348,7 @@ export function updateIngestionType(
   spec: IngestionSpec,
   comboType: IngestionComboType,
 ): IngestionSpec {
-  let [ingestionType, firehoseType] = comboType.split(':');
-  if (ingestionType === 'index') ingestionType = 'index_parallel';
+  const [ingestionType, inputSourceType] = comboType.split(':');
   const ioAndTuningConfigType = ingestionTypeToIoAndTuningConfigType(
     ingestionType as IngestionType,
   );
@@ -2248,11 +2358,11 @@ export function updateIngestionType(
   newSpec = deepSet(newSpec, 'ioConfig.type', ioAndTuningConfigType);
   newSpec = deepSet(newSpec, 'tuningConfig.type', ioAndTuningConfigType);
 
-  if (firehoseType) {
-    newSpec = deepSet(newSpec, 'ioConfig.firehose', { type: firehoseType });
+  if (inputSourceType) {
+    newSpec = deepSet(newSpec, 'ioConfig.inputSource', { type: inputSourceType });
 
-    if (firehoseType === 'local') {
-      newSpec = deepSet(newSpec, 'ioConfig.firehose.filter', '*');
+    if (inputSourceType === 'local') {
+      newSpec = deepSet(newSpec, 'ioConfig.inputSource.filter', '*');
     }
   }
 
@@ -2272,65 +2382,64 @@ export function updateIngestionType(
     newSpec = deepSet(newSpec, 'dataSchema.granularitySpec', granularitySpec);
   }
 
+  if (!deepGet(spec, 'dataSchema.timestampSpec')) {
+    newSpec = deepSet(newSpec, 'dataSchema.timestampSpec', getDummyTimestampSpec());
+  }
+
+  if (!deepGet(spec, 'dataSchema.dimensionsSpec')) {
+    newSpec = deepSet(newSpec, 'dataSchema.dimensionsSpec', {});
+  }
+
   return newSpec;
 }
 
-export function fillParser(spec: IngestionSpec, sampleData: string[]): IngestionSpec {
-  const firehoseType = deepGet(spec, 'ioConfig.firehose.type');
+export function fillInputFormat(spec: IngestionSpec, sampleData: string[]): IngestionSpec {
+  const inputFormat = guessInputFormat(sampleData);
+  if (!inputFormat) return spec;
 
-  if (firehoseType === 'sql') {
-    return deepSet(spec, 'dataSchema.parser', { type: 'map' });
-  }
-
-  if (firehoseType === 'ingestSegment') {
-    return deepSet(spec, 'dataSchema.parser', {
-      type: 'string',
-      parseSpec: { format: 'timeAndDims' },
-    });
-  }
-
-  const parseSpec = guessParseSpec(sampleData);
-  if (!parseSpec) return spec;
-
-  return deepSet(spec, 'dataSchema.parser', { type: 'string', parseSpec });
+  return deepSet(spec, 'ioConfig.inputFormat', inputFormat);
 }
 
-function guessParseSpec(sampleData: string[]): ParseSpec | undefined {
+function guessInputFormat(sampleData: string[]): InputFormat | undefined {
   const sampleDatum = sampleData[0];
   if (!sampleDatum) return;
 
   if (sampleDatum.startsWith('{') && sampleDatum.endsWith('}')) {
-    return parseSpecFromFormat('json');
+    return inputFormatFromType('json');
   }
 
   if (sampleDatum.split('\t').length > 3) {
-    return parseSpecFromFormat('tsv', !/\t\d+\t/.test(sampleDatum));
+    return inputFormatFromType('tsv', !/\t\d+\t/.test(sampleDatum));
   }
 
   if (sampleDatum.split(',').length > 3) {
-    return parseSpecFromFormat('csv', !/,\d+,/.test(sampleDatum));
+    return inputFormatFromType('csv', !/,\d+,/.test(sampleDatum));
   }
 
-  return parseSpecFromFormat('regex');
+  if (sampleDatum.startsWith('PAR1')) {
+    return inputFormatFromType('parquet');
+  }
+
+  if (sampleDatum.startsWith('ORC')) {
+    return inputFormatFromType('orc');
+  }
+
+  return inputFormatFromType('regex');
 }
 
-function parseSpecFromFormat(format: string, hasHeaderRow?: boolean): ParseSpec {
-  const parseSpec: ParseSpec = {
-    format,
-    timestampSpec: {},
-    dimensionsSpec: {},
-  };
+function inputFormatFromType(type: string, findColumnsFromHeader?: boolean): InputFormat {
+  const inputFormat: InputFormat = { type };
 
-  if (format === 'regex') {
-    parseSpec.pattern = '(.*)';
-    parseSpec.columns = ['column1'];
+  if (type === 'regex') {
+    inputFormat.pattern = '(.*)';
+    inputFormat.columns = ['column1'];
   }
 
-  if (typeof hasHeaderRow === 'boolean') {
-    parseSpec.hasHeaderRow = hasHeaderRow;
+  if (typeof findColumnsFromHeader === 'boolean') {
+    inputFormat.findColumnsFromHeader = findColumnsFromHeader;
   }
 
-  return parseSpec;
+  return inputFormat;
 }
 
 export type DruidFilter = Record<string, any>;
@@ -2437,3 +2546,55 @@ const FILTER_FORM_FIELDS: Field<DruidFilter>[] = [
 export function getFilterFormFields() {
   return FILTER_FORM_FIELDS;
 }
+
+export function upgradeSpec(spec: any): any {
+  if (deepGet(spec, 'ioConfig.firehose')) {
+    switch (deepGet(spec, 'ioConfig.firehose.type')) {
+      case 'static-s3':
+        deepSet(spec, 'ioConfig.firehose.type', 's3');
+        break;
+
+      case 'static-google-blobstore':
+        deepSet(spec, 'ioConfig.firehose.type', 'google');
+        break;
+    }
+
+    spec = deepMove(spec, 'ioConfig.firehose', 'ioConfig.inputSource');
+    spec = deepMove(spec, 'dataSchema.parser.parseSpec.timestampSpec', 'dataSchema.timestampSpec');
+    spec = deepMove(
+      spec,
+      'dataSchema.parser.parseSpec.dimensionsSpec',
+      'dataSchema.dimensionsSpec',
+    );
+    spec = deepMove(spec, 'dataSchema.parser.parseSpec', 'ioConfig.inputFormat');
+    spec = deepDelete(spec, 'dataSchema.parser');
+    spec = deepMove(spec, 'ioConfig.inputFormat.format', 'ioConfig.inputFormat.type');
+  }
+  return spec;
+}
+
+export function downgradeSpec(spec: any): any {
+  if (deepGet(spec, 'ioConfig.inputSource')) {
+    spec = deepMove(spec, 'ioConfig.inputFormat.type', 'ioConfig.inputFormat.format');
+    spec = deepSet(spec, 'dataSchema.parser', { type: 'string' });
+    spec = deepMove(spec, 'ioConfig.inputFormat', 'dataSchema.parser.parseSpec');
+    spec = deepMove(
+      spec,
+      'dataSchema.dimensionsSpec',
+      'dataSchema.parser.parseSpec.dimensionsSpec',
+    );
+    spec = deepMove(spec, 'dataSchema.timestampSpec', 'dataSchema.parser.parseSpec.timestampSpec');
+    spec = deepMove(spec, 'ioConfig.inputSource', 'ioConfig.firehose');
+
+    switch (deepGet(spec, 'ioConfig.firehose.type')) {
+      case 's3':
+        deepSet(spec, 'ioConfig.firehose.type', 'static-s3');
+        break;
+
+      case 'google':
+        deepSet(spec, 'ioConfig.firehose.type', 'static-google-blobstore');
+        break;
+    }
+  }
+  return spec;
+}
diff --git a/web-console/src/utils/load-rule.ts b/web-console/src/utils/load-rule.ts
index 47068b84dd5..ecd90306397 100644
--- a/web-console/src/utils/load-rule.ts
+++ b/web-console/src/utils/load-rule.ts
@@ -16,17 +16,22 @@
  * limitations under the License.
  */
 
+import { deepMove, deepSet } from './object-change';
+
+export type RuleType =
+  | 'loadForever'
+  | 'loadByInterval'
+  | 'loadByPeriod'
+  | 'dropForever'
+  | 'dropByInterval'
+  | 'dropByPeriod'
+  | 'dropBeforeByPeriod'
+  | 'broadcastForever'
+  | 'broadcastByInterval'
+  | 'broadcastByPeriod';
+
 export interface Rule {
-  type:
-    | 'loadForever'
-    | 'loadByInterval'
-    | 'loadByPeriod'
-    | 'dropForever'
-    | 'dropByInterval'
-    | 'dropByPeriod'
-    | 'broadcastForever'
-    | 'broadcastByInterval'
-    | 'broadcastByPeriod';
+  type: RuleType;
   interval?: string;
   period?: string;
   includeFuture?: boolean;
@@ -34,83 +39,97 @@ export interface Rule {
   colocatedDataSources?: string[];
 }
 
-export type LoadType = 'load' | 'drop' | 'broadcast';
-export type TimeType = 'Forever' | 'ByInterval' | 'ByPeriod';
-
 export class RuleUtil {
-  static shouldIncludeFuture(rule: Rule): boolean {
-    if (rule.includeFuture !== false) {
-      return (
-        rule.type === 'loadByPeriod' ||
-        rule.type === 'dropByPeriod' ||
-        rule.type === 'broadcastByPeriod'
-      );
-    }
-    return false;
-  }
+  static TYPES: RuleType[] = [
+    'loadForever',
+    'loadByInterval',
+    'loadByPeriod',
+    'dropForever',
+    'dropByInterval',
+    'dropByPeriod',
+    'dropBeforeByPeriod',
+    'broadcastForever',
+    'broadcastByInterval',
+    'broadcastByPeriod',
+  ];
 
   static ruleToString(rule: Rule): string {
-    return (
-      rule.type +
-      (rule.period ? `(${rule.period})` : '') +
-      (rule.interval ? `(${rule.interval})` : '') +
-      (RuleUtil.shouldIncludeFuture(rule) ? `(includeFuture)` : '')
-    );
+    return [
+      rule.type,
+      rule.period ? `(${rule.period}${rule.includeFuture ? `+future` : ''})` : '',
+      rule.interval ? `(${rule.interval})` : '',
+    ].join('');
   }
 
-  static getLoadType(rule: Rule): LoadType {
-    const m = rule.type.match(/^(load|drop|broadcast)(\w+)$/);
-    if (!m) throw new Error(`unknown rule type: '${rule.type}'`);
-    return m[1] as any;
-  }
+  static changeRuleType(rule: Rule, type: RuleType): Rule {
+    const newRule = deepSet(rule, 'type', type);
 
-  static getTimeType(rule: Rule): TimeType {
-    const m = rule.type.match(/^(load|drop|broadcast)(\w+)$/);
-    if (!m) throw new Error(`unknown rule type: '${rule.type}'`);
-    return m[2] as any;
-  }
+    if (RuleUtil.hasPeriod(newRule)) {
+      if (!newRule.period) newRule.period = 'P1M';
+    } else {
+      delete newRule.period;
+      delete newRule.includeFuture;
+    }
+
+    if (RuleUtil.hasInterval(newRule)) {
+      if (!newRule.interval) newRule.interval = '2010-01-01/2020-01-01';
+    } else {
+      delete newRule.interval;
+    }
+
+    if (RuleUtil.hasTieredReplicants(newRule)) {
+      if (!newRule.tieredReplicants) newRule.tieredReplicants = { _default_tier: 2 };
+    } else {
+      delete newRule.tieredReplicants;
+    }
+
+    if (!RuleUtil.hasColocatedDataSources(newRule)) delete newRule.colocatedDataSources;
 
-  static changeLoadType(rule: Rule, loadType: LoadType): Rule {
-    const newRule = Object.assign({}, rule, { type: loadType + RuleUtil.getTimeType(rule) });
-    if (loadType !== 'load') delete newRule.tieredReplicants;
-    if (loadType !== 'broadcast') delete newRule.colocatedDataSources;
     return newRule;
   }
 
-  static changeTimeType(rule: Rule, timeType: TimeType): Rule {
-    const newRule = Object.assign({}, rule, { type: RuleUtil.getLoadType(rule) + timeType });
-    if (timeType !== 'ByPeriod') delete newRule.period;
-    if (timeType !== 'ByInterval') delete newRule.interval;
-    return newRule;
+  static hasPeriod(rule: Rule): boolean {
+    return rule.type.endsWith('ByPeriod');
   }
 
   static changePeriod(rule: Rule, period: string): Rule {
-    return Object.assign({}, rule, { period });
+    return deepSet(rule, 'period', period);
+  }
+
+  static hasIncludeFuture(rule: Rule): boolean {
+    return RuleUtil.hasPeriod(rule) && rule.type !== 'dropBeforeByPeriod';
   }
 
   static changeIncludeFuture(rule: Rule, includeFuture: boolean): Rule {
-    return Object.assign({}, rule, { includeFuture });
+    return deepSet(rule, 'includeFuture', includeFuture);
+  }
+
+  static hasInterval(rule: Rule): boolean {
+    return rule.type.endsWith('ByInterval');
   }
 
   static changeInterval(rule: Rule, interval: string): Rule {
-    return Object.assign({}, rule, { interval });
+    return deepSet(rule, 'interval', interval);
   }
 
-  static changeTier(rule: Rule, oldTier: string, newTier: string): Rule {
-    const newRule = Object.assign({}, rule);
-    newRule.tieredReplicants = Object.assign({}, newRule.tieredReplicants);
-    newRule.tieredReplicants[newTier] = newRule.tieredReplicants[oldTier];
-    delete newRule.tieredReplicants[oldTier];
-    return newRule;
+  static hasTieredReplicants(rule: Rule): boolean {
+    return rule.type.startsWith('load');
   }
 
-  static changeTierReplication(rule: Rule, tier: string, replication: number): Rule {
-    const newRule = Object.assign({}, rule);
-    newRule.tieredReplicants = Object.assign({}, newRule.tieredReplicants, { [tier]: replication });
-    return newRule;
+  static renameTieredReplicants(rule: Rule, oldTier: string, newTier: string): Rule {
+    return deepMove(rule, `tieredReplicants.${oldTier}`, `tieredReplicants.${newTier}`);
+  }
+
+  static addTieredReplicant(rule: Rule, tier: string, replication: number): Rule {
+    const newTieredReplicants = deepSet(rule.tieredReplicants || {}, tier, replication);
+    return deepSet(rule, 'tieredReplicants', newTieredReplicants);
+  }
+
+  static hasColocatedDataSources(rule: Rule): boolean {
+    return rule.type.startsWith('broadcast');
   }
 
   static changeColocatedDataSources(rule: Rule, colocatedDataSources: string[]): Rule {
-    return Object.assign({}, rule, { colocatedDataSources });
+    return deepSet(rule, 'colocatedDataSources', colocatedDataSources);
   }
 }
diff --git a/web-console/src/utils/object-change.ts b/web-console/src/utils/object-change.ts
index 2ff608fd52d..83b7cceaa2b 100644
--- a/web-console/src/utils/object-change.ts
+++ b/web-console/src/utils/object-change.ts
@@ -111,6 +111,16 @@ export function deepDelete<T extends Record<string, any>>(value: T, path: string
   return valueCopy;
 }
 
+export function deepMove<T extends Record<string, any>>(
+  value: T,
+  fromPath: string,
+  toPath: string,
+): T {
+  value = deepSet(value, toPath, deepGet(value, fromPath));
+  value = deepDelete(value, fromPath);
+  return value;
+}
+
 export function deepExtend<T extends Record<string, any>>(target: T, diff: Record<string, any>): T {
   if (typeof target !== 'object') throw new TypeError(`Invalid target`);
   if (typeof diff !== 'object') throw new TypeError(`Invalid diff`);
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index 83c0401a81c..8b188ecb307 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -22,25 +22,24 @@ import { getDruidErrorMessage, queryDruidRune } from './druid-query';
 import { alphanumericCompare, filterMap, sortWithPrefixSuffix } from './general';
 import {
   DimensionsSpec,
-  getEmptyTimestampSpec,
+  getDummyTimestampSpec,
   getSpecType,
   IngestionSpec,
+  IngestionType,
+  InputFormat,
   IoConfig,
   isColumnTimestampSpec,
-  isIngestSegment,
+  isDruidSource,
   MetricSpec,
-  Parser,
-  ParseSpec,
+  TimestampSpec,
   Transform,
   TransformSpec,
+  upgradeSpec,
 } from './ingestion-spec';
-import { deepGet, deepSet, whitelistKeys } from './object-change';
-
-const MS_IN_HOUR = 60 * 60 * 1000;
+import { deepGet, deepSet } from './object-change';
 
 const SAMPLER_URL = `/druid/indexer/v1/sampler`;
 const BASE_SAMPLER_CONFIG: SamplerConfig = {
-  // skipCache: true,
   numRows: 500,
   timeoutMs: 15000,
 };
@@ -54,25 +53,23 @@ export interface SampleSpec {
 export interface SamplerConfig {
   numRows?: number;
   timeoutMs?: number;
-  cacheKey?: string;
-  skipCache?: boolean;
 }
 
 export interface SampleResponse {
-  cacheKey?: string;
   data: SampleEntry[];
 }
 
+export type CacheRows = Record<string, any>[];
+
 export interface SampleResponseWithExtraInfo extends SampleResponse {
   queryGranularity?: any;
-  timestampSpec?: any;
   rollup?: boolean;
   columns?: Record<string, any>;
   aggregators?: Record<string, any>;
 }
 
 export interface SampleEntry {
-  raw: string;
+  input: Record<string, any>;
   parsed?: Record<string, any>;
   unparseable?: boolean;
   error?: string;
@@ -101,12 +98,35 @@ function dedupe(xs: string[]): string[] {
   });
 }
 
-type SamplerType = 'index' | 'kafka' | 'kinesis';
+export function getCacheRowsFromSampleResponse(
+  sampleResponse: SampleResponse,
+  useParsed = false,
+): CacheRows {
+  const key = useParsed ? 'parsed' : 'input';
+  return filterMap(sampleResponse.data, d => d[key]).slice(0, 20);
+}
 
-export function getSamplerType(spec: IngestionSpec): SamplerType {
-  const specType = getSpecType(spec);
-  if (specType === 'kafka' || specType === 'kinesis') return specType;
-  return 'index';
+export function applyCache(sampleSpec: SampleSpec, cacheRows: CacheRows) {
+  if (!cacheRows) return sampleSpec;
+
+  // If this is already an inline spec there is nothing to do
+  if (deepGet(sampleSpec, 'spec.ioConfig.inputSource.type') === 'inline') return sampleSpec;
+
+  // Make the spec into an inline json spec
+  sampleSpec = deepSet(sampleSpec, 'type', 'index');
+  sampleSpec = deepSet(sampleSpec, 'spec.type', 'index');
+  sampleSpec = deepSet(sampleSpec, 'spec.ioConfig.type', 'index');
+  sampleSpec = deepSet(sampleSpec, 'spec.ioConfig.inputSource', {
+    type: 'inline',
+    data: cacheRows.map(r => JSON.stringify(r)).join('\n'),
+  });
+
+  const flattenSpec = deepGet(sampleSpec, 'spec.ioConfig.inputFormat.flattenSpec');
+  const inputFormat: InputFormat = { type: 'json' };
+  if (flattenSpec) inputFormat.flattenSpec = flattenSpec;
+  sampleSpec = deepSet(sampleSpec, 'spec.ioConfig.inputFormat', inputFormat);
+
+  return sampleSpec;
 }
 
 export function headerFromSampleResponse(
@@ -140,7 +160,7 @@ export function headerAndRowsFromSampleResponse(
   };
 }
 
-export async function getOverlordModules(): Promise<string[]> {
+export async function getProxyOverlordModules(): Promise<string[]> {
   let statusResp: any;
   try {
     statusResp = await axios.get(`/proxy/overlord/status`);
@@ -155,6 +175,8 @@ export async function postToSampler(
   sampleSpec: SampleSpec,
   forStr: string,
 ): Promise<SampleResponse> {
+  sampleSpec = fixSamplerTypes(sampleSpec);
+
   let sampleResp: any;
   try {
     sampleResp = await axios.post(`${SAMPLER_URL}?for=${forStr}`, sampleSpec);
@@ -169,77 +191,72 @@ export type SampleStrategy = 'start' | 'end';
 
 function makeSamplerIoConfig(
   ioConfig: IoConfig,
-  samplerType: SamplerType,
+  specType: IngestionType,
   sampleStrategy: SampleStrategy,
 ): IoConfig {
-  ioConfig = deepSet(ioConfig || {}, 'type', samplerType);
-  if (samplerType === 'kafka') {
+  ioConfig = deepSet(ioConfig || {}, 'type', specType);
+  if (specType === 'kafka') {
     ioConfig = deepSet(ioConfig, 'useEarliestOffset', sampleStrategy === 'start');
-  } else if (samplerType === 'kinesis') {
+  } else if (specType === 'kinesis') {
     ioConfig = deepSet(ioConfig, 'useEarliestSequenceNumber', sampleStrategy === 'start');
   }
   return ioConfig;
 }
 
 /**
- * This function scopes down the interval of an ingestSegment firehose for the data sampler
- * this is needed because the ingestSegment firehose gets the interval you are sampling over,
- * looks up the corresponding segments and segment locations from metadata store, downloads
- * every segment from deep storage to disk, and then maps all the segments into memory;
- * and this happens in the constructor before the timer thread is even created meaning the sampler
- * will time out on a larger interval.
- * This is essentially a workaround for https://github.com/apache/incubator-druid/issues/8448
- * @param ioConfig The IO Config to scope down the interval of
+  This is a hack to deal with the fact that the sampler can not deal with the index_parallel type
  */
-export async function scopeDownIngestSegmentFirehoseIntervalIfNeeded(
-  ioConfig: IoConfig,
-): Promise<IoConfig> {
-  if (deepGet(ioConfig, 'firehose.type') !== 'ingestSegment') return ioConfig;
-  const interval = deepGet(ioConfig, 'firehose.interval');
-  const intervalParts = interval.split('/');
-  const start = new Date(intervalParts[0]);
-  if (isNaN(start.valueOf())) throw new Error(`could not decode interval start`);
-  const end = new Date(intervalParts[1]);
-  if (isNaN(end.valueOf())) throw new Error(`could not decode interval end`);
+function fixSamplerTypes(sampleSpec: SampleSpec): SampleSpec {
+  let samplerType: string = getSpecType(sampleSpec.spec);
+  if (samplerType === 'index_parallel') {
+    samplerType = 'index';
+  }
 
-  // Less than or equal to 1 hour so there is no need to adjust intervals
-  if (Math.abs(end.valueOf() - start.valueOf()) <= MS_IN_HOUR) return ioConfig;
+  sampleSpec = deepSet(sampleSpec, 'type', samplerType);
+  sampleSpec = deepSet(sampleSpec, 'spec.type', samplerType);
+  sampleSpec = deepSet(sampleSpec, 'spec.ioConfig.type', samplerType);
+  sampleSpec = deepSet(sampleSpec, 'spec.tuningConfig.type', samplerType);
+  return sampleSpec;
+}
 
-  const dataSourceMetadataResponse = await queryDruidRune({
-    queryType: 'dataSourceMetadata',
-    dataSource: deepGet(ioConfig, 'firehose.dataSource'),
-  });
+function cleanupQueryGranularity(queryGranularity: any): any {
+  let queryGranularityType = deepGet(queryGranularity, 'type');
+  if (typeof queryGranularityType !== 'string') return queryGranularity;
+  queryGranularityType = queryGranularityType.toUpperCase();
 
-  const maxIngestedEventTime = new Date(
-    deepGet(dataSourceMetadataResponse, '0.result.maxIngestedEventTime'),
-  );
+  const knownGranularity = [
+    'NONE',
+    'SECOND',
+    'MINUTE',
+    'HOUR',
+    'DAY',
+    'WEEK',
+    'MONTH',
+    'YEAR',
+  ].includes(queryGranularityType);
 
-  // If invalid maxIngestedEventTime do nothing
-  if (isNaN(maxIngestedEventTime.valueOf())) return ioConfig;
-
-  // If maxIngestedEventTime is before the start of the interval do nothing
-  if (maxIngestedEventTime < start) return ioConfig;
-
-  const newEnd = maxIngestedEventTime < end ? maxIngestedEventTime : end;
-  const newStart = new Date(newEnd.valueOf() - MS_IN_HOUR); // Set start to 1 hour ago
-
-  return deepSet(
-    ioConfig,
-    'firehose.interval',
-    `${newStart.toISOString()}/${newEnd.toISOString()}`,
-  );
+  return knownGranularity ? queryGranularityType : queryGranularity;
 }
 
 export async function sampleForConnect(
   spec: IngestionSpec,
   sampleStrategy: SampleStrategy,
 ): Promise<SampleResponseWithExtraInfo> {
-  const samplerType = getSamplerType(spec);
-  const ioConfig: IoConfig = await scopeDownIngestSegmentFirehoseIntervalIfNeeded(
-    makeSamplerIoConfig(deepGet(spec, 'ioConfig'), samplerType, sampleStrategy),
+  const samplerType = getSpecType(spec);
+  let ioConfig: IoConfig = makeSamplerIoConfig(
+    deepGet(spec, 'ioConfig'),
+    samplerType,
+    sampleStrategy,
   );
 
-  const ingestSegmentMode = isIngestSegment(spec);
+  const reingestMode = isDruidSource(spec);
+  if (!reingestMode) {
+    ioConfig = deepSet(ioConfig, 'inputFormat', {
+      type: 'regex',
+      pattern: '(.*)',
+      columns: ['raw'],
+    });
+  }
 
   const sampleSpec: SampleSpec = {
     type: samplerType,
@@ -248,16 +265,8 @@ export async function sampleForConnect(
       ioConfig,
       dataSchema: {
         dataSource: 'sample',
-        parser: {
-          type: 'string',
-          parseSpec: {
-            format: 'regex',
-            pattern: '(.*)',
-            columns: ['a'],
-            dimensionsSpec: {},
-            timestampSpec: getEmptyTimestampSpec(),
-          },
-        },
+        timestampSpec: getDummyTimestampSpec(),
+        dimensionsSpec: {},
       },
     } as any,
     samplerConfig: BASE_SAMPLER_CONFIG,
@@ -267,11 +276,11 @@ export async function sampleForConnect(
 
   if (!samplerResponse.data.length) return samplerResponse;
 
-  if (ingestSegmentMode) {
+  if (reingestMode) {
     const segmentMetadataResponse = await queryDruidRune({
       queryType: 'segmentMetadata',
-      dataSource: deepGet(ioConfig, 'firehose.dataSource'),
-      intervals: [deepGet(ioConfig, 'firehose.interval')],
+      dataSource: deepGet(ioConfig, 'inputSource.dataSource'),
+      intervals: [deepGet(ioConfig, 'inputSource.interval')],
       merge: true,
       lenientAggregatorMerge: true,
       analysisTypes: ['timestampSpec', 'queryGranularity', 'aggregators', 'rollup'],
@@ -279,8 +288,9 @@ export async function sampleForConnect(
 
     if (Array.isArray(segmentMetadataResponse) && segmentMetadataResponse.length === 1) {
       const segmentMetadataResponse0 = segmentMetadataResponse[0];
-      samplerResponse.queryGranularity = segmentMetadataResponse0.queryGranularity;
-      samplerResponse.timestampSpec = segmentMetadataResponse0.timestampSpec;
+      samplerResponse.queryGranularity = cleanupQueryGranularity(
+        segmentMetadataResponse0.queryGranularity,
+      );
       samplerResponse.rollup = segmentMetadataResponse0.rollup;
       samplerResponse.columns = segmentMetadataResponse0.columns;
       samplerResponse.aggregators = segmentMetadataResponse0.aggregators;
@@ -295,35 +305,26 @@ export async function sampleForConnect(
 export async function sampleForParser(
   spec: IngestionSpec,
   sampleStrategy: SampleStrategy,
-  cacheKey: string | undefined,
 ): Promise<SampleResponse> {
-  const samplerType = getSamplerType(spec);
-  const ioConfig: IoConfig = await scopeDownIngestSegmentFirehoseIntervalIfNeeded(
-    makeSamplerIoConfig(deepGet(spec, 'ioConfig'), samplerType, sampleStrategy),
+  const samplerType = getSpecType(spec);
+  const ioConfig: IoConfig = makeSamplerIoConfig(
+    deepGet(spec, 'ioConfig'),
+    samplerType,
+    sampleStrategy,
   );
-  const parser: Parser = deepGet(spec, 'dataSchema.parser') || {};
 
   const sampleSpec: SampleSpec = {
     type: samplerType,
     spec: {
       type: samplerType,
-      ioConfig: deepSet(ioConfig, 'type', samplerType),
+      ioConfig,
       dataSchema: {
         dataSource: 'sample',
-        parser: {
-          type: parser.type,
-          parseSpec: (parser.parseSpec
-            ? Object.assign({}, parser.parseSpec, {
-                dimensionsSpec: {},
-                timestampSpec: getEmptyTimestampSpec(),
-              })
-            : undefined) as any,
-        },
+        timestampSpec: getDummyTimestampSpec(),
+        dimensionsSpec: {},
       },
     },
-    samplerConfig: Object.assign({}, BASE_SAMPLER_CONFIG, {
-      cacheKey,
-    }),
+    samplerConfig: BASE_SAMPLER_CONFIG,
   };
 
   return postToSampler(sampleSpec, 'parser');
@@ -331,17 +332,10 @@ export async function sampleForParser(
 
 export async function sampleForTimestamp(
   spec: IngestionSpec,
-  sampleStrategy: SampleStrategy,
-  cacheKey: string | undefined,
+  cacheRows: CacheRows,
 ): Promise<SampleResponse> {
-  const samplerType = getSamplerType(spec);
-  const ioConfig: IoConfig = await scopeDownIngestSegmentFirehoseIntervalIfNeeded(
-    makeSamplerIoConfig(deepGet(spec, 'ioConfig'), samplerType, sampleStrategy),
-  );
-  const parser: Parser = deepGet(spec, 'dataSchema.parser') || {};
-  const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || {};
-  const timestampSpec: ParseSpec =
-    deepGet(spec, 'dataSchema.parser.parseSpec.timestampSpec') || getEmptyTimestampSpec();
+  const samplerType = getSpecType(spec);
+  const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec');
   const columnTimestampSpec = isColumnTimestampSpec(timestampSpec);
 
   // First do a query with a static timestamp spec
@@ -349,26 +343,20 @@ export async function sampleForTimestamp(
     type: samplerType,
     spec: {
       type: samplerType,
-      ioConfig: deepSet(ioConfig, 'type', samplerType),
+      ioConfig: deepGet(spec, 'ioConfig'),
       dataSchema: {
         dataSource: 'sample',
-        parser: {
-          type: parser.type,
-          parseSpec: (parser.parseSpec
-            ? Object.assign({}, parseSpec, {
-                dimensionsSpec: {},
-                timestampSpec: columnTimestampSpec ? getEmptyTimestampSpec() : timestampSpec,
-              })
-            : undefined) as any,
-        },
+        dimensionsSpec: {},
+        timestampSpec: columnTimestampSpec ? getDummyTimestampSpec() : timestampSpec,
       },
     },
-    samplerConfig: Object.assign({}, BASE_SAMPLER_CONFIG, {
-      cacheKey,
-    }),
+    samplerConfig: BASE_SAMPLER_CONFIG,
   };
 
-  const sampleColumns = await postToSampler(sampleSpecColumns, 'timestamp-columns');
+  const sampleColumns = await postToSampler(
+    applyCache(sampleSpecColumns, cacheRows),
+    'timestamp-columns',
+  );
 
   // If we are not parsing a column then there is nothing left to do
   if (!columnTimestampSpec) return sampleColumns;
@@ -379,28 +367,19 @@ export async function sampleForTimestamp(
     type: samplerType,
     spec: {
       type: samplerType,
-      ioConfig: deepSet(ioConfig, 'type', samplerType),
+      ioConfig: deepGet(spec, 'ioConfig'),
       dataSchema: {
         dataSource: 'sample',
-        parser: {
-          type: parser.type,
-          parseSpec: Object.assign({}, parseSpec, {
-            dimensionsSpec: {},
-          }),
-        },
+        dimensionsSpec: {},
+        timestampSpec,
       },
     },
-    samplerConfig: Object.assign({}, BASE_SAMPLER_CONFIG, {
-      cacheKey: sampleColumns.cacheKey || cacheKey,
-    }),
+    samplerConfig: BASE_SAMPLER_CONFIG,
   };
 
-  const sampleTime = await postToSampler(sampleSpec, 'timestamp-time');
+  const sampleTime = await postToSampler(applyCache(sampleSpec, cacheRows), 'timestamp-time');
 
-  if (
-    sampleTime.cacheKey !== sampleColumns.cacheKey ||
-    sampleTime.data.length !== sampleColumns.data.length
-  ) {
+  if (sampleTime.data.length !== sampleColumns.data.length) {
     // If the two responses did not come from the same cache (or for some reason have different lengths) then
     // just return the one with the parsed time column.
     return sampleTime;
@@ -420,16 +399,11 @@ export async function sampleForTimestamp(
 
 export async function sampleForTransform(
   spec: IngestionSpec,
-  sampleStrategy: SampleStrategy,
-  cacheKey: string | undefined,
+  cacheRows: CacheRows,
 ): Promise<SampleResponse> {
-  const samplerType = getSamplerType(spec);
-  const ioConfig: IoConfig = await scopeDownIngestSegmentFirehoseIntervalIfNeeded(
-    makeSamplerIoConfig(deepGet(spec, 'ioConfig'), samplerType, sampleStrategy),
-  );
-  const parser: Parser = deepGet(spec, 'dataSchema.parser') || {};
-  const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || {};
-  const parserColumns: string[] = deepGet(parseSpec, 'columns') || [];
+  const samplerType = getSpecType(spec);
+  const inputFormatColumns: string[] = deepGet(spec, 'ioConfig.inputFormat.columns') || [];
+  const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec');
   const transforms: Transform[] = deepGet(spec, 'dataSchema.transformSpec.transforms') || [];
 
   // Extra step to simulate auto detecting dimension with transforms
@@ -439,29 +413,26 @@ export async function sampleForTransform(
       type: samplerType,
       spec: {
         type: samplerType,
-        ioConfig: deepSet(ioConfig, 'type', samplerType),
+        ioConfig: deepGet(spec, 'ioConfig'),
         dataSchema: {
           dataSource: 'sample',
-          parser: {
-            type: parser.type,
-            parseSpec: Object.assign({}, parseSpec, {
-              dimensionsSpec: {},
-            }),
-          },
+          timestampSpec,
+          dimensionsSpec: {},
         },
       },
-      samplerConfig: Object.assign({}, BASE_SAMPLER_CONFIG, {
-        cacheKey,
-      }),
+      samplerConfig: BASE_SAMPLER_CONFIG,
     };
 
-    const sampleResponseHack = await postToSampler(sampleSpecHack, 'transform-pre');
+    const sampleResponseHack = await postToSampler(
+      applyCache(sampleSpecHack, cacheRows),
+      'transform-pre',
+    );
 
     specialDimensionSpec.dimensions = dedupe(
       headerFromSampleResponse(
         sampleResponseHack,
         '__time',
-        ['__time'].concat(parserColumns),
+        ['__time'].concat(inputFormatColumns),
       ).concat(transforms.map(t => t.name)),
     );
   }
@@ -470,40 +441,29 @@ export async function sampleForTransform(
     type: samplerType,
     spec: {
       type: samplerType,
-      ioConfig: deepSet(ioConfig, 'type', samplerType),
+      ioConfig: deepGet(spec, 'ioConfig'),
       dataSchema: {
         dataSource: 'sample',
-        parser: {
-          type: parser.type,
-          parseSpec: Object.assign({}, parseSpec, {
-            dimensionsSpec: specialDimensionSpec, // Hack Hack Hack
-          }),
-        },
+        timestampSpec,
+        dimensionsSpec: specialDimensionSpec, // Hack Hack Hack
         transformSpec: {
           transforms,
         },
       },
     },
-    samplerConfig: Object.assign({}, BASE_SAMPLER_CONFIG, {
-      cacheKey,
-    }),
+    samplerConfig: BASE_SAMPLER_CONFIG,
   };
 
-  return postToSampler(sampleSpec, 'transform');
+  return postToSampler(applyCache(sampleSpec, cacheRows), 'transform');
 }
 
 export async function sampleForFilter(
   spec: IngestionSpec,
-  sampleStrategy: SampleStrategy,
-  cacheKey: string | undefined,
+  cacheRows: CacheRows,
 ): Promise<SampleResponse> {
-  const samplerType = getSamplerType(spec);
-  const ioConfig: IoConfig = await scopeDownIngestSegmentFirehoseIntervalIfNeeded(
-    makeSamplerIoConfig(deepGet(spec, 'ioConfig'), samplerType, sampleStrategy),
-  );
-  const parser: Parser = deepGet(spec, 'dataSchema.parser') || {};
-  const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || {};
-  const parserColumns: string[] = deepGet(parser, 'columns') || [];
+  const samplerType = getSpecType(spec);
+  const inputFormatColumns: string[] = deepGet(spec, 'ioConfig.inputFormat.columns') || [];
+  const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec');
   const transforms: Transform[] = deepGet(spec, 'dataSchema.transformSpec.transforms') || [];
   const filter: any = deepGet(spec, 'dataSchema.transformSpec.filter');
 
@@ -514,29 +474,26 @@ export async function sampleForFilter(
       type: samplerType,
       spec: {
         type: samplerType,
-        ioConfig: deepSet(ioConfig, 'type', samplerType),
+        ioConfig: deepGet(spec, 'ioConfig'),
         dataSchema: {
           dataSource: 'sample',
-          parser: {
-            type: parser.type,
-            parseSpec: Object.assign({}, parseSpec, {
-              dimensionsSpec: {},
-            }),
-          },
+          timestampSpec,
+          dimensionsSpec: {},
         },
       },
-      samplerConfig: Object.assign({}, BASE_SAMPLER_CONFIG, {
-        cacheKey,
-      }),
+      samplerConfig: BASE_SAMPLER_CONFIG,
     };
 
-    const sampleResponseHack = await postToSampler(sampleSpecHack, 'filter-pre');
+    const sampleResponseHack = await postToSampler(
+      applyCache(sampleSpecHack, cacheRows),
+      'filter-pre',
+    );
 
     specialDimensionSpec.dimensions = dedupe(
       headerFromSampleResponse(
         sampleResponseHack,
         '__time',
-        ['__time'].concat(parserColumns),
+        ['__time'].concat(inputFormatColumns),
       ).concat(transforms.map(t => t.name)),
     );
   }
@@ -545,41 +502,32 @@ export async function sampleForFilter(
     type: samplerType,
     spec: {
       type: samplerType,
-      ioConfig: deepSet(ioConfig, 'type', samplerType),
+      ioConfig: deepGet(spec, 'ioConfig'),
       dataSchema: {
         dataSource: 'sample',
-        parser: {
-          type: parser.type,
-          parseSpec: Object.assign({}, parseSpec, {
-            dimensionsSpec: specialDimensionSpec, // Hack Hack Hack
-          }),
-        },
+        timestampSpec,
+        dimensionsSpec: specialDimensionSpec, // Hack Hack Hack
         transformSpec: {
           transforms,
           filter,
         },
       },
     },
-    samplerConfig: Object.assign({}, BASE_SAMPLER_CONFIG, {
-      cacheKey,
-    }),
+    samplerConfig: BASE_SAMPLER_CONFIG,
   };
 
-  return postToSampler(sampleSpec, 'filter');
+  return postToSampler(applyCache(sampleSpec, cacheRows), 'filter');
 }
 
 export async function sampleForSchema(
   spec: IngestionSpec,
-  sampleStrategy: SampleStrategy,
-  cacheKey: string | undefined,
+  cacheRows: CacheRows,
 ): Promise<SampleResponse> {
-  const samplerType = getSamplerType(spec);
-  const ioConfig: IoConfig = await scopeDownIngestSegmentFirehoseIntervalIfNeeded(
-    makeSamplerIoConfig(deepGet(spec, 'ioConfig'), samplerType, sampleStrategy),
-  );
-  const parser: Parser = deepGet(spec, 'dataSchema.parser') || {};
+  const samplerType = getSpecType(spec);
+  const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec');
   const transformSpec: TransformSpec =
     deepGet(spec, 'dataSchema.transformSpec') || ({} as TransformSpec);
+  const dimensionsSpec: DimensionsSpec = deepGet(spec, 'dataSchema.dimensionsSpec');
   const metricsSpec: MetricSpec[] = deepGet(spec, 'dataSchema.metricsSpec') || [];
   const queryGranularity: string =
     deepGet(spec, 'dataSchema.granularitySpec.queryGranularity') || 'NONE';
@@ -588,56 +536,49 @@ export async function sampleForSchema(
     type: samplerType,
     spec: {
       type: samplerType,
-      ioConfig: deepSet(ioConfig, 'type', samplerType),
+      ioConfig: deepGet(spec, 'ioConfig'),
       dataSchema: {
         dataSource: 'sample',
-        parser: whitelistKeys(parser, ['type', 'parseSpec']) as Parser,
+        timestampSpec,
         transformSpec,
-        metricsSpec,
         granularitySpec: {
           queryGranularity,
         },
+        dimensionsSpec,
+        metricsSpec,
       },
     },
-    samplerConfig: Object.assign({}, BASE_SAMPLER_CONFIG, {
-      cacheKey,
-    }),
+    samplerConfig: BASE_SAMPLER_CONFIG,
   };
 
-  return postToSampler(sampleSpec, 'schema');
+  return postToSampler(applyCache(sampleSpec, cacheRows), 'schema');
 }
 
 export async function sampleForExampleManifests(
   exampleManifestUrl: string,
 ): Promise<ExampleManifest[]> {
-  const sampleSpec: SampleSpec = {
-    type: 'index',
+  const exampleSpec: SampleSpec = {
+    type: 'index_parallel',
     spec: {
-      type: 'index',
+      type: 'index_parallel',
       ioConfig: {
-        type: 'index',
-        firehose: { type: 'http', uris: [exampleManifestUrl] },
+        type: 'index_parallel',
+        inputSource: { type: 'http', uris: [exampleManifestUrl] },
+        inputFormat: { type: 'tsv', findColumnsFromHeader: true },
       },
       dataSchema: {
         dataSource: 'sample',
-        parser: {
-          type: 'string',
-          parseSpec: {
-            format: 'tsv',
-            timestampSpec: {
-              column: 'timestamp',
-              missingValue: '2010-01-01T00:00:00Z',
-            },
-            dimensionsSpec: {},
-            hasHeaderRow: true,
-          },
+        timestampSpec: {
+          column: 'timestamp',
+          missingValue: '2010-01-01T00:00:00Z',
         },
+        dimensionsSpec: {},
       },
     },
-    samplerConfig: { numRows: 50, timeoutMs: 10000, skipCache: true },
+    samplerConfig: { numRows: 50, timeoutMs: 10000 },
   };
 
-  const exampleData = await postToSampler(sampleSpec, 'example-manifest');
+  const exampleData = await postToSampler(exampleSpec, 'example-manifest');
 
   return filterMap(exampleData.data, datum => {
     const parsed = datum.parsed;
@@ -658,7 +599,7 @@ export async function sampleForExampleManifests(
       return {
         name: parsed.name,
         description: parsed.description,
-        spec,
+        spec: upgradeSpec(spec),
       };
     } else {
       return;
diff --git a/web-console/src/utils/spec-utils.ts b/web-console/src/utils/spec-utils.ts
index 8e0b465dabd..2842118ba99 100644
--- a/web-console/src/utils/spec-utils.ts
+++ b/web-console/src/utils/spec-utils.ts
@@ -26,10 +26,10 @@ export function computeFlattenPathsForData(
   exprType: ExprType,
   arrayHandling: ArrayHandling,
 ): FlattenField[] {
-  return computeFlattenExprsForData(data, exprType, arrayHandling).map((expr, i) => {
+  return computeFlattenExprsForData(data, exprType, arrayHandling).map(expr => {
     return {
+      name: expr.replace(/^\$?\./, ''),
       type: exprType,
-      name: `expr_${i}`,
       expr,
     };
   });
diff --git a/web-console/src/utils/utils.spec.ts b/web-console/src/utils/utils.spec.ts
index dda3c73b3ae..b4d31ce0e61 100644
--- a/web-console/src/utils/utils.spec.ts
+++ b/web-console/src/utils/utils.spec.ts
@@ -25,27 +25,20 @@ import {
   updateSchemaWithSample,
 } from './druid-type';
 import { IngestionSpec } from './ingestion-spec';
-import {
-  getSamplerType,
-  headerFromSampleResponse,
-  sampleForConnect,
-  sampleForExampleManifests,
-  sampleForFilter,
-  sampleForParser,
-  sampleForSchema,
-  sampleForTimestamp,
-  sampleForTransform,
-} from './sampler';
+import { applyCache, headerFromSampleResponse } from './sampler';
 
 describe('test-utils', () => {
-  const ingestionSpec = {
+  const ingestionSpec: IngestionSpec = {
     type: 'index_parallel',
     ioConfig: {
       type: 'index_parallel',
-      firehose: {
+      inputSource: {
         type: 'http',
         uris: ['https://static.imply.io/data/wikipedia.json.gz'],
       },
+      inputFormat: {
+        type: 'json',
+      },
     },
     tuningConfig: {
       type: 'index_parallel',
@@ -57,71 +50,122 @@ describe('test-utils', () => {
         segmentGranularity: 'DAY',
         queryGranularity: 'HOUR',
       },
-      parser: {
-        type: 'string',
-        parseSpec: {
-          format: 'json',
-          timestampSpec: {
-            column: 'timestamp',
-            format: 'iso',
-          },
-          dimensionsSpec: {},
-        },
+      timestampSpec: {
+        column: 'timestamp',
+        format: 'iso',
       },
+      dimensionsSpec: {},
     },
   };
-  it('spec-utils getSamplerType', () => {
-    expect(getSamplerType(ingestionSpec as IngestionSpec)).toMatchInlineSnapshot(`"index"`);
-  });
+
+  // const cacheRows: CacheRows = [{ make: 'Honda', model: 'Civic' }, { make: 'BMW', model: 'M3' }];
+
   it('spec-utils headerFromSampleResponse', () => {
-    expect(headerFromSampleResponse({ cacheKey: 'abc123', data: [] })).toMatchInlineSnapshot(
-      `Array []`,
-    );
+    expect(headerFromSampleResponse({ data: [{ input: { a: 1 }, parsed: { a: 1 } }] }))
+      .toMatchInlineSnapshot(`
+      Array [
+        "a",
+      ]
+    `);
   });
-  it('spec-utils sampleForParser', () => {
+
+  it('spec-utils applyCache', () => {
     expect(
-      sampleForParser(ingestionSpec as IngestionSpec, 'start', 'abc123'),
-    ).toMatchInlineSnapshot(`Promise {}`);
-  });
-  it('spec-utils SampleSpec', () => {
-    expect(sampleForConnect(ingestionSpec as IngestionSpec, 'start')).toMatchInlineSnapshot(
-      `Promise {}`,
-    );
-  });
-  it('spec-utils sampleForTimestamp', () => {
-    expect(
-      sampleForTimestamp(ingestionSpec as IngestionSpec, 'start', 'abc123'),
-    ).toMatchInlineSnapshot(`Promise {}`);
-  });
-  it('spec-utils sampleForTransform', () => {
-    expect(
-      sampleForTransform(ingestionSpec as IngestionSpec, 'start', 'abc123'),
-    ).toMatchInlineSnapshot(`Promise {}`);
-  });
-  it('spec-utils sampleForFilter', () => {
-    expect(
-      sampleForFilter(ingestionSpec as IngestionSpec, 'start', 'abc123'),
-    ).toMatchInlineSnapshot(`Promise {}`);
-  });
-  it('spec-utils sampleForSchema', () => {
-    expect(
-      sampleForSchema(ingestionSpec as IngestionSpec, 'start', 'abc123'),
-    ).toMatchInlineSnapshot(`Promise {}`);
-  });
-  it('spec-utils sampleForExampleManifests', () => {
-    expect(sampleForExampleManifests('abc123')).toMatchInlineSnapshot(`Promise {}`);
+      applyCache(
+        {
+          type: 'index_parallel',
+          spec: ingestionSpec,
+          samplerConfig: {
+            numRows: 500,
+            timeoutMs: 15000,
+          },
+        },
+        [{ make: 'Honda', model: 'Accord' }, { make: 'Toyota', model: 'Prius' }],
+      ),
+    ).toMatchInlineSnapshot(`
+      Object {
+        "samplerConfig": Object {
+          "numRows": 500,
+          "timeoutMs": 15000,
+        },
+        "spec": Object {
+          "dataSchema": Object {
+            "dataSource": "wikipedia",
+            "dimensionsSpec": Object {},
+            "granularitySpec": Object {
+              "queryGranularity": "HOUR",
+              "segmentGranularity": "DAY",
+              "type": "uniform",
+            },
+            "timestampSpec": Object {
+              "column": "timestamp",
+              "format": "iso",
+            },
+          },
+          "ioConfig": Object {
+            "inputFormat": Object {
+              "type": "json",
+            },
+            "inputSource": Object {
+              "data": "{\\"make\\":\\"Honda\\",\\"model\\":\\"Accord\\"}
+      {\\"make\\":\\"Toyota\\",\\"model\\":\\"Prius\\"}",
+              "type": "inline",
+            },
+            "type": "index",
+          },
+          "tuningConfig": Object {
+            "type": "index_parallel",
+          },
+          "type": "index",
+        },
+        "type": "index",
+      }
+    `);
   });
+
+  // it('spec-utils sampleForParser', async () => {
+  //   expect(await sampleForParser(ingestionSpec, 'start', 'abc123')).toMatchInlineSnapshot(
+  //     `Promise {}`,
+  //   );
+  // });
+  //
+  // it('spec-utils SampleSpec', async () => {
+  //   expect(await sampleForConnect(ingestionSpec, 'start')).toMatchInlineSnapshot(`Promise {}`);
+  // });
+  //
+  // it('spec-utils sampleForTimestamp', async () => {
+  //   expect(await sampleForTimestamp(ingestionSpec, 'start', cacheRows)).toMatchInlineSnapshot();
+  // });
+  //
+  // it('spec-utils sampleForTransform', async () => {
+  //   expect(await sampleForTransform(ingestionSpec, 'start', cacheRows)).toMatchInlineSnapshot();
+  // });
+  //
+  // it('spec-utils sampleForFilter', async () => {
+  //   expect(await sampleForFilter(ingestionSpec, 'start', cacheRows)).toMatchInlineSnapshot();
+  // });
+  //
+  // it('spec-utils sampleForSchema', async () => {
+  //   expect(await sampleForSchema(ingestionSpec, 'start', cacheRows)).toMatchInlineSnapshot();
+  // });
+  //
+  // it('spec-utils sampleForExampleManifests', async () => {
+  //   expect(await sampleForExampleManifests('some url')).toMatchInlineSnapshot();
+  // });
 });
 
 describe('druid-type.ts', () => {
-  const ingestionSpec = {
+  const ingestionSpec: IngestionSpec = {
     type: 'index_parallel',
     ioConfig: {
       type: 'index_parallel',
-      firehose: {
+      inputSource: {
         type: 'http',
         uris: ['https://static.imply.io/data/wikipedia.json.gz'],
       },
+      inputFormat: {
+        type: 'json',
+      },
     },
     tuningConfig: {
       type: 'index_parallel',
@@ -133,27 +177,24 @@ describe('druid-type.ts', () => {
         segmentGranularity: 'DAY',
         queryGranularity: 'HOUR',
       },
-      parser: {
-        type: 'string',
-        parseSpec: {
-          format: 'json',
-          timestampSpec: {
-            column: 'timestamp',
-            format: 'iso',
-          },
-          dimensionsSpec: {},
-        },
+      timestampSpec: {
+        column: 'timestamp',
+        format: 'iso',
       },
+      dimensionsSpec: {},
     },
   };
-  it('spec-utils getSamplerType', () => {
+
+  it('spec-utils guessTypeFromSample', () => {
     expect(guessTypeFromSample([])).toMatchInlineSnapshot(`"string"`);
   });
+
   it('spec-utils getColumnTypeFromHeaderAndRows', () => {
     expect(
       getColumnTypeFromHeaderAndRows({ header: ['header'], rows: [] }, 'header'),
     ).toMatchInlineSnapshot(`"string"`);
   });
+
   it('spec-utils getDimensionSpecs', () => {
     expect(getDimensionSpecs({ header: ['header'], rows: [] }, true)).toMatchInlineSnapshot(`
       Array [
@@ -161,6 +202,7 @@ describe('druid-type.ts', () => {
       ]
     `);
   });
+
   it('spec-utils getMetricSecs', () => {
     expect(getMetricSecs({ header: ['header'], rows: [] })).toMatchInlineSnapshot(`
       Array [
@@ -171,18 +213,19 @@ describe('druid-type.ts', () => {
       ]
     `);
   });
+
   it('spec-utils updateSchemaWithSample', () => {
     expect(
-      updateSchemaWithSample(
-        ingestionSpec as IngestionSpec,
-        { header: ['header'], rows: [] },
-        'specific',
-        true,
-      ),
+      updateSchemaWithSample(ingestionSpec, { header: ['header'], rows: [] }, 'specific', true),
     ).toMatchInlineSnapshot(`
       Object {
         "dataSchema": Object {
           "dataSource": "wikipedia",
+          "dimensionsSpec": Object {
+            "dimensions": Array [
+              "header",
+            ],
+          },
           "granularitySpec": Object {
             "queryGranularity": "HOUR",
             "rollup": true,
@@ -195,24 +238,16 @@ describe('druid-type.ts', () => {
               "type": "count",
             },
           ],
-          "parser": Object {
-            "parseSpec": Object {
-              "dimensionsSpec": Object {
-                "dimensions": Array [
-                  "header",
-                ],
-              },
-              "format": "json",
-              "timestampSpec": Object {
-                "column": "timestamp",
-                "format": "iso",
-              },
-            },
-            "type": "string",
+          "timestampSpec": Object {
+            "column": "timestamp",
+            "format": "iso",
           },
         },
         "ioConfig": Object {
-          "firehose": Object {
+          "inputFormat": Object {
+            "type": "json",
+          },
+          "inputSource": Object {
             "type": "http",
             "uris": Array [
               "https://static.imply.io/data/wikipedia.json.gz",
@@ -232,9 +267,11 @@ describe('druid-query.ts', () => {
   it('spec-utils parseHtmlError', () => {
     expect(parseHtmlError('<div></div>')).toMatchInlineSnapshot(`undefined`);
   });
+
   it('spec-utils parseHtmlError', () => {
     expect(getDruidErrorMessage({})).toMatchInlineSnapshot(`undefined`);
   });
+
   it('spec-utils parseQueryPlan', () => {
     expect(parseQueryPlan('start')).toMatchInlineSnapshot(`"start"`);
   });
diff --git a/web-console/src/views/ingestion-view/__snapshots__/ingestion-view.spec.tsx.snap b/web-console/src/views/ingestion-view/__snapshots__/ingestion-view.spec.tsx.snap
index 40b9bfa3803..f66b2722bee 100644
--- a/web-console/src/views/ingestion-view/__snapshots__/ingestion-view.spec.tsx.snap
+++ b/web-console/src/views/ingestion-view/__snapshots__/ingestion-view.spec.tsx.snap
@@ -514,7 +514,7 @@ exports[`tasks view matches snapshot 1`] = `
         filtered={
           Array [
             Object {
-              "id": "task_id",
+              "id": "group_id",
               "value": "test",
             },
             Object {
diff --git a/web-console/src/views/ingestion-view/ingestion-view.spec.tsx b/web-console/src/views/ingestion-view/ingestion-view.spec.tsx
index 24599d6609b..fa8c867e18f 100644
--- a/web-console/src/views/ingestion-view/ingestion-view.spec.tsx
+++ b/web-console/src/views/ingestion-view/ingestion-view.spec.tsx
@@ -28,7 +28,7 @@ describe('tasks view', () => {
     const taskView = shallow(
       <IngestionView
         openDialog={'test'}
-        taskId={'test'}
+        taskGroupId={'test'}
         datasourceId={'datasource'}
         goToDatasource={() => {}}
         goToQuery={() => {}}
diff --git a/web-console/src/views/ingestion-view/ingestion-view.tsx b/web-console/src/views/ingestion-view/ingestion-view.tsx
index add6adb5d8c..f8a201cc7f7 100644
--- a/web-console/src/views/ingestion-view/ingestion-view.tsx
+++ b/web-console/src/views/ingestion-view/ingestion-view.tsx
@@ -102,7 +102,7 @@ interface TaskQueryResultRow {
 }
 
 export interface IngestionViewProps {
-  taskId: string | undefined;
+  taskGroupId: string | undefined;
   datasourceId: string | undefined;
   openDialog: string | undefined;
   goToDatasource: (datasource: string) => void;
@@ -216,7 +216,7 @@ ORDER BY "rank" DESC, "created_time" DESC`;
     super(props, context);
 
     const taskFilter: Filter[] = [];
-    if (props.taskId) taskFilter.push({ id: 'task_id', value: props.taskId });
+    if (props.taskGroupId) taskFilter.push({ id: 'group_id', value: props.taskGroupId });
     if (props.datasourceId) taskFilter.push({ id: 'datasource', value: props.datasourceId });
 
     const supervisorFilter: Filter[] = [];
diff --git a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
index ff459c7e179..b3b6bd0de0e 100644
--- a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
+++ b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
@@ -59,12 +59,12 @@ exports[`load data view matches snapshot 1`] = `
     </div>
     <div
       className="step-section"
-      key="Transform and configure schema"
+      key="Transform data and configure schema"
     >
       <div
         className="step-nav-l1"
       >
-        Transform and configure schema
+        Transform data and configure schema
       </div>
       <Blueprint3.ButtonGroup
         className="step-nav-l2"
diff --git a/web-console/src/views/load-data-view/filter-table/filter-table.spec.tsx b/web-console/src/views/load-data-view/filter-table/filter-table.spec.tsx
index 968c20923cd..0c635b17947 100644
--- a/web-console/src/views/load-data-view/filter-table/filter-table.spec.tsx
+++ b/web-console/src/views/load-data-view/filter-table/filter-table.spec.tsx
@@ -27,7 +27,7 @@ describe('filter table', () => {
       header: ['c1'],
       rows: [
         {
-          raw: `{"c1":"hello"}`,
+          input: { c1: 'hello' },
           parsed: { c1: 'hello' },
         },
       ],
diff --git a/web-console/src/views/load-data-view/load-data-view.spec.tsx b/web-console/src/views/load-data-view/load-data-view.spec.tsx
index 7ff02daf658..166a7e1ba04 100644
--- a/web-console/src/views/load-data-view/load-data-view.spec.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.spec.tsx
@@ -23,7 +23,7 @@ import { LoadDataView } from './load-data-view';
 
 describe('load data view', () => {
   it('matches snapshot', () => {
-    const loadDataView = shallow(<LoadDataView goToTask={() => {}} />);
+    const loadDataView = shallow(<LoadDataView goToIngestion={() => {}} />);
     expect(loadDataView).toMatchSnapshot();
   });
 });
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index dbc6ee5fa03..4e733459fe8 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -63,12 +63,13 @@ import {
   LocalStorageKeys,
   localStorageSet,
   parseJson,
+  pluralIfNeeded,
   QueryState,
 } from '../../utils';
 import { NUMERIC_TIME_FORMATS, possibleDruidFormatForValues } from '../../utils/druid-time';
 import { updateSchemaWithSample } from '../../utils/druid-type';
 import {
-  changeParallel,
+  adjustIngestionSpec,
   DimensionMode,
   DimensionSpec,
   DimensionsSpec,
@@ -76,21 +77,21 @@ import {
   EMPTY_ARRAY,
   EMPTY_OBJECT,
   fillDataSourceNameIfNeeded,
-  fillParser,
+  fillInputFormat,
   FlattenField,
+  getConstantTimestampSpec,
   getDimensionMode,
   getDimensionSpecFormFields,
-  getEmptyTimestampSpec,
   getFilterFormFields,
   getFlattenFieldFormFields,
   getIngestionComboType,
   getIngestionDocLink,
   getIngestionImage,
   getIngestionTitle,
+  getInputFormatFormFields,
   getIoConfigFormFields,
   getIoConfigTuningFormFields,
   getMetricSpecFormFields,
-  getParseSpecFormFields,
   getPartitionRelatedTuningSpecFormFields,
   getRequiredModule,
   getRollup,
@@ -99,36 +100,35 @@ import {
   getTransformFormFields,
   getTuningSpecFormFields,
   GranularitySpec,
-  hasParallelAbility,
   IngestionComboTypeWithExtra,
   IngestionSpec,
+  InputFormat,
+  inputFormatCanFlatten,
   invalidIoConfig,
   invalidTuningConfig,
   IoConfig,
   isColumnTimestampSpec,
+  isDruidSource,
   isEmptyIngestionSpec,
-  isIngestSegment,
-  isParallel,
   issueWithIoConfig,
-  issueWithParser,
   isTask,
   joinFilter,
   MAX_INLINE_DATA_LENGTH,
   MetricSpec,
   normalizeSpec,
-  Parser,
-  ParseSpec,
-  parseSpecHasFlatten,
   splitFilter,
   TimestampSpec,
   Transform,
   TuningConfig,
   updateIngestionType,
+  upgradeSpec,
 } from '../../utils/ingestion-spec';
 import { deepDelete, deepGet, deepSet } from '../../utils/object-change';
 import {
+  CacheRows,
   ExampleManifest,
-  getOverlordModules,
+  getCacheRowsFromSampleResponse,
+  getProxyOverlordModules,
   HeaderAndRows,
   headerAndRowsFromSampleResponse,
   SampleEntry,
@@ -163,7 +163,9 @@ import {
 import './load-data-view.scss';
 
 function showRawLine(line: SampleEntry): string {
-  const raw = line.raw;
+  if (!line.parsed) return 'No parse';
+  const raw = line.parsed.raw;
+  if (typeof raw !== 'string') return String(raw);
   if (raw.includes('\n')) {
     return `[Multi-line row, length: ${raw.length}]`;
   }
@@ -173,12 +175,17 @@ function showRawLine(line: SampleEntry): string {
   return raw;
 }
 
+function showDruidLine(line: SampleEntry): string {
+  if (!line.parsed) return 'No parse';
+  return `Druid row: ${JSON.stringify(line.parsed)}`;
+}
+
 function showBlankLine(line: SampleEntry): string {
   return line.parsed ? `[Row: ${JSON.stringify(line.parsed)}]` : '[Binary data]';
 }
 
 function getTimestampSpec(headerAndRows: HeaderAndRows | null): TimestampSpec {
-  if (!headerAndRows) return getEmptyTimestampSpec();
+  if (!headerAndRows) return getConstantTimestampSpec();
 
   const timestampSpecs = filterMap(headerAndRows.header, sampleHeader => {
     const possibleFormat = possibleDruidFormatForValues(
@@ -195,7 +202,7 @@ function getTimestampSpec(headerAndRows: HeaderAndRows | null): TimestampSpec {
     timestampSpecs.find(ts => /time/i.test(ts.column)) || // Use a suggestion that has time in the name if possible
     timestampSpecs.find(ts => !NUMERIC_TIME_FORMATS.includes(ts.format)) || // Use a suggestion that is not numeric
     timestampSpecs[0] || // Fall back to the first one
-    getEmptyTimestampSpec() // Ok, empty it is...
+    getConstantTimestampSpec() // Ok, empty it is...
   );
 }
 
@@ -230,7 +237,7 @@ const STEPS: Step[] = [
 
 const SECTIONS: { name: string; steps: Step[] }[] = [
   { name: 'Connect and parse raw data', steps: ['welcome', 'connect', 'parser', 'timestamp'] },
-  { name: 'Transform and configure schema', steps: ['transform', 'filter', 'schema'] },
+  { name: 'Transform data and configure schema', steps: ['transform', 'filter', 'schema'] },
   { name: 'Tune parameters', steps: ['partition', 'tuning', 'publish'] },
   { name: 'Verify and submit', steps: ['spec'] },
 ];
@@ -254,14 +261,14 @@ export interface LoadDataViewProps {
   initSupervisorId?: string;
   initTaskId?: string;
   exampleManifestsUrl?: string;
-  goToTask: (taskId: string | undefined, supervisor?: string) => void;
+  goToIngestion: (taskGroupId: string | undefined, supervisor?: string) => void;
 }
 
 export interface LoadDataViewState {
   step: Step;
   spec: IngestionSpec;
   specPreview: IngestionSpec;
-  cacheKey?: string;
+  cacheRows?: CacheRows;
   // dialogs / modals
   continueToSpec: boolean;
   showResetConfirm: boolean;
@@ -340,7 +347,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       columnFilter: '',
       specialColumnsOnly: false,
 
-      // for firehose
+      // for inputSource
       inputQueryState: QueryState.INIT,
 
       // for parser
@@ -396,7 +403,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   async getOverlordModules() {
     let overlordModules: string[];
     try {
-      overlordModules = await getOverlordModules();
+      overlordModules = await getProxyOverlordModules();
     } catch (e) {
       AppToaster.show({
         message: `Failed to get overlord modules: ${e.message}`,
@@ -410,23 +417,27 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   }
 
   isStepEnabled(step: Step): boolean {
-    const { spec } = this.state;
+    const { spec, cacheRows } = this.state;
+    const druidSource = isDruidSource(spec);
     const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
-    const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
 
     switch (step) {
       case 'connect':
         return Boolean(spec.type);
 
       case 'parser':
+        return Boolean(!druidSource && spec.type && !issueWithIoConfig(ioConfig));
+
       case 'timestamp':
+        return Boolean(!druidSource && cacheRows);
+
       case 'transform':
       case 'filter':
       case 'schema':
       case 'partition':
       case 'tuning':
       case 'publish':
-        return Boolean(spec.type && !issueWithIoConfig(ioConfig) && !issueWithParser(parser));
+        return Boolean(cacheRows);
 
       default:
         return true;
@@ -439,7 +450,13 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
   private updateSpec = (newSpec: IngestionSpec) => {
     newSpec = normalizeSpec(newSpec);
-    this.setState({ spec: newSpec, specPreview: newSpec });
+    newSpec = upgradeSpec(newSpec);
+    newSpec = adjustIngestionSpec(newSpec);
+    const deltaState: Partial<LoadDataViewState> = { spec: newSpec, specPreview: newSpec };
+    if (!deepGet(newSpec, 'ioConfig.type')) {
+      deltaState.cacheRows = undefined;
+    }
+    this.setState(deltaState as LoadDataViewState);
     localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSON.stringify(newSpec));
   };
 
@@ -606,22 +623,13 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     );
   }
 
-  renderNextBar(options: {
-    nextStep?: Step;
-    disabled?: boolean;
-    onNextStep?: () => void;
-    onPrevStep?: () => void;
-    prevLabel?: string;
-  }) {
-    const { disabled, onNextStep, onPrevStep, prevLabel } = options;
+  renderNextBar(options: { nextStep?: Step; disabled?: boolean; onNextStep?: () => void }) {
+    const { disabled, onNextStep } = options;
     const { step } = this.state;
     const nextStep = options.nextStep || STEPS[STEPS.indexOf(step) + 1] || STEPS[0];
 
     return (
       <div className="next-bar">
-        {onPrevStep && (
-          <Button className="prev" icon={IconNames.UNDO} text={prevLabel} onClick={onPrevStep} />
-        )}
         <Button
           text={`Next: ${VIEW_TITLE[nextStep]}`}
           rightIcon={IconNames.ARROW_RIGHT}
@@ -695,13 +703,13 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         <div className="main bp3-input">
           {this.renderIngestionCard('kafka')}
           {this.renderIngestionCard('kinesis')}
-          {this.renderIngestionCard('index:static-s3')}
-          {this.renderIngestionCard('index:static-google-blobstore')}
-          {this.renderIngestionCard('index:hdfs')}
-          {this.renderIngestionCard('index:ingestSegment')}
-          {this.renderIngestionCard('index:http')}
-          {this.renderIngestionCard('index:local')}
-          {this.renderIngestionCard('index:inline')}
+          {this.renderIngestionCard('index_parallel:s3')}
+          {this.renderIngestionCard('index_parallel:google')}
+          {this.renderIngestionCard('index_parallel:hdfs')}
+          {this.renderIngestionCard('index_parallel:druid')}
+          {this.renderIngestionCard('index_parallel:http')}
+          {this.renderIngestionCard('index_parallel:local')}
+          {this.renderIngestionCard('index_parallel:inline')}
           {exampleManifestsUrl && this.renderIngestionCard('example', noExamples)}
           {this.renderIngestionCard('other')}
         </div>
@@ -727,7 +735,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     if (issue) return issue;
 
     switch (selectedComboType) {
-      case 'index:http':
+      case 'index_parallel:http':
         return (
           <>
             <p>Load data accessible through HTTP(s).</p>
@@ -738,7 +746,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           </>
         );
 
-      case 'index:local':
+      case 'index_parallel:local':
         return (
           <>
             <p>
@@ -752,7 +760,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           </>
         );
 
-      case 'index:ingestSegment':
+      case 'index_parallel:druid':
         return (
           <>
             <p>Reindex data from existing Druid segments.</p>
@@ -763,20 +771,20 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           </>
         );
 
-      case 'index:inline':
+      case 'index_parallel:inline':
         return (
           <>
             <p>Ingest a small amount of data directly from the clipboard.</p>
           </>
         );
 
-      case 'index:static-s3':
+      case 'index_parallel:s3':
         return <p>Load text based data from Amazon S3.</p>;
 
-      case 'index:static-google-blobstore':
+      case 'index_parallel:google':
         return <p>Load text based data from the Google Blobstore.</p>;
 
-      case 'index:hdfs':
+      case 'index_parallel:hdfs':
         return <p>Load text based data from HDFS.</p>;
 
       case 'kafka':
@@ -811,20 +819,20 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   }
 
   renderWelcomeStepControls(): JSX.Element | undefined {
-    const { goToTask } = this.props;
+    const { goToIngestion } = this.props;
     const { spec, selectedComboType, exampleManifests } = this.state;
 
     const issue = this.selectedIngestionTypeIssue();
     if (issue) return;
 
     switch (selectedComboType) {
-      case 'index:http':
-      case 'index:local':
-      case 'index:ingestSegment':
-      case 'index:inline':
-      case 'index:static-s3':
-      case 'index:static-google-blobstore':
-      case 'index:hdfs':
+      case 'index_parallel:http':
+      case 'index_parallel:local':
+      case 'index_parallel:druid':
+      case 'index_parallel:inline':
+      case 'index_parallel:s3':
+      case 'index_parallel:google':
+      case 'index_parallel:hdfs':
       case 'kafka':
       case 'kinesis':
         return (
@@ -861,7 +869,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 text="Submit supervisor"
                 rightIcon={IconNames.ARROW_RIGHT}
                 intent={Intent.PRIMARY}
-                onClick={() => goToTask(undefined, 'supervisor')}
+                onClick={() => goToIngestion(undefined, 'supervisor')}
               />
             </FormGroup>
             <FormGroup>
@@ -869,7 +877,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 text="Submit task"
                 rightIcon={IconNames.ARROW_RIGHT}
                 intent={Intent.PRIMARY}
-                onClick={() => goToTask(undefined, 'task')}
+                onClick={() => goToIngestion(undefined, 'task')}
               />
             </FormGroup>
           </>
@@ -951,7 +959,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
 
     let issue: string | undefined;
-    if (issueWithIoConfig(ioConfig)) {
+    if (issueWithIoConfig(ioConfig, true)) {
       issue = `IoConfig not ready, ${issueWithIoConfig(ioConfig)}`;
     }
 
@@ -976,17 +984,21 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       return;
     }
 
-    this.setState({
-      cacheKey: sampleResponse.cacheKey,
+    const deltaState: Partial<LoadDataViewState> = {
       inputQueryState: new QueryState({ data: sampleResponse }),
-    });
+    };
+    if (isDruidSource(spec)) {
+      deltaState.cacheRows = getCacheRowsFromSampleResponse(sampleResponse, true);
+    }
+    this.setState(deltaState as LoadDataViewState);
   }
 
   renderConnectStep() {
     const { specPreview: spec, inputQueryState, sampleStrategy } = this.state;
     const specType = getSpecType(spec);
     const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
-    const inlineMode = deepGet(spec, 'ioConfig.firehose.type') === 'inline';
+    const inlineMode = deepGet(spec, 'ioConfig.inputSource.type') === 'inline';
+    const druidSource = isDruidSource(spec);
 
     let mainFill: JSX.Element | string = '';
     if (inlineMode) {
@@ -994,10 +1006,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         <TextArea
           className="inline-data"
           placeholder="Paste your data here"
-          value={deepGet(spec, 'ioConfig.firehose.data')}
+          value={deepGet(spec, 'ioConfig.inputSource.data')}
           onChange={(e: any) => {
             const stringValue = e.target.value.substr(0, MAX_INLINE_DATA_LENGTH);
-            this.updateSpecPreview(deepSet(spec, 'ioConfig.firehose.data', stringValue));
+            this.updateSpecPreview(deepSet(spec, 'ioConfig.inputSource.data', stringValue));
           }}
         />
       );
@@ -1017,15 +1029,17 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       mainFill = (
         <TextArea
           className="raw-lines"
+          readOnly
           value={
             inputData.length
-              ? (inputData.every(l => !l.raw)
+              ? (inputData.every(l => !l.parsed)
                   ? inputData.map(showBlankLine)
+                  : druidSource
+                  ? inputData.map(showDruidLine)
                   : inputData.map(showRawLine)
                 ).join('\n')
               : 'No data returned from sampler'
           }
-          readOnly
         />
       );
     }
@@ -1070,7 +1084,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               />
             </FormGroup>
           )}
-          {deepGet(spec, 'ioConfig.firehose.type') === 'local' && (
+          {deepGet(spec, 'ioConfig.inputSource.type') === 'local' && (
             <FormGroup>
               <Callout intent={Intent.WARNING}>
                 This path must be available on the local filesystem of all Druid services.
@@ -1092,25 +1106,23 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         </div>
         {this.renderNextBar({
           disabled: !inputQueryState.data,
+          nextStep: druidSource ? 'transform' : 'parser',
           onNextStep: () => {
             if (!inputQueryState.data) return;
             const inputData = inputQueryState.data;
 
-            if (isIngestSegment(spec)) {
-              let newSpec = fillParser(spec, []);
+            if (druidSource) {
+              let newSpec = fillInputFormat(spec, []);
+
+              newSpec = deepSet(newSpec, 'dataSchema.timestampSpec', {
+                column: '__time',
+                format: 'iso',
+              });
 
               if (typeof inputData.rollup === 'boolean') {
                 newSpec = deepSet(newSpec, 'dataSchema.granularitySpec.rollup', inputData.rollup);
               }
 
-              if (inputData.timestampSpec) {
-                newSpec = deepSet(
-                  newSpec,
-                  'dataSchema.parser.parseSpec.timestampSpec',
-                  inputData.timestampSpec,
-                );
-              }
-
               if (inputData.queryGranularity) {
                 newSpec = deepSet(
                   newSpec,
@@ -1123,7 +1135,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 const aggregators = inputData.aggregators || {};
                 newSpec = deepSet(
                   newSpec,
-                  'dataSchema.parser.parseSpec.dimensionsSpec.dimensions',
+                  'dataSchema.dimensionsSpec.dimensions',
                   Object.keys(inputData.columns)
                     .filter(k => k !== '__time' && !aggregators[k])
                     .map(k => ({
@@ -1145,7 +1157,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             } else {
               this.updateSpec(
                 fillDataSourceNameIfNeeded(
-                  fillParser(spec, inputQueryState.data.data.map(l => l.raw)),
+                  fillInputFormat(
+                    spec,
+                    filterMap(inputQueryState.data.data, l =>
+                      l.parsed ? l.parsed.raw : undefined,
+                    ),
+                  ),
                 ),
               );
             }
@@ -1158,16 +1175,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   // ==================================================================
 
   async queryForParser(initRun = false) {
-    const { spec, sampleStrategy, cacheKey } = this.state;
+    const { spec, sampleStrategy } = this.state;
     const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
-    const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
-    const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
+    const inputFormatColumns: string[] =
+      deepGet(spec, 'ioConfig.inputFormat.columns') || EMPTY_ARRAY;
 
     let issue: string | undefined;
     if (issueWithIoConfig(ioConfig)) {
       issue = `IoConfig not ready, ${issueWithIoConfig(ioConfig)}`;
-    } else if (issueWithParser(parser)) {
-      issue = `Parser not ready, ${issueWithParser(parser)}`;
     }
 
     if (issue) {
@@ -1183,7 +1198,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
     let sampleResponse: SampleResponse;
     try {
-      sampleResponse = await sampleForParser(spec, sampleStrategy, cacheKey);
+      sampleResponse = await sampleForParser(spec, sampleStrategy);
     } catch (e) {
       this.setState({
         parserQueryState: new QueryState({ error: e.message }),
@@ -1192,9 +1207,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     }
 
     this.setState({
-      cacheKey: sampleResponse.cacheKey,
+      cacheRows: getCacheRowsFromSampleResponse(sampleResponse),
       parserQueryState: new QueryState({
-        data: headerAndRowsFromSampleResponse(sampleResponse, '__time', parserColumns),
+        data: headerAndRowsFromSampleResponse(sampleResponse, '__time', inputFormatColumns),
       }),
     });
   }
@@ -1207,11 +1222,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       parserQueryState,
       selectedFlattenField,
     } = this.state;
-    const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || EMPTY_OBJECT;
+    const inputFormat: InputFormat = deepGet(spec, 'ioConfig.inputFormat') || EMPTY_OBJECT;
     const flattenFields: FlattenField[] =
-      deepGet(spec, 'dataSchema.parser.parseSpec.flattenSpec.fields') || EMPTY_ARRAY;
+      deepGet(spec, 'ioConfig.inputFormat.flattenSpec.fields') || EMPTY_ARRAY;
 
-    const canFlatten = parseSpec.format === 'json';
+    const canFlatten = inputFormatCanFlatten(inputFormat);
 
     let mainFill: JSX.Element | string = '';
     if (parserQueryState.isInit()) {
@@ -1254,10 +1269,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       );
     }
 
-    let sugestedFlattenFields: FlattenField[] | null = null;
+    let sugestedFlattenFields: FlattenField[] | undefined;
     if (canFlatten && !flattenFields.length && parserQueryState.data) {
       sugestedFlattenFields = computeFlattenPathsForData(
-        filterMap(parserQueryState.data.rows, r => parseJson(r.raw)),
+        filterMap(parserQueryState.data.rows, r => r.input),
         'path',
         'ignore-arrays',
       );
@@ -1292,32 +1307,28 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           {!selectedFlattenField && (
             <>
               <AutoForm
-                fields={getParseSpecFormFields()}
-                model={parseSpec}
-                onChange={p =>
-                  this.updateSpecPreview(deepSet(spec, 'dataSchema.parser.parseSpec', p))
-                }
+                fields={getInputFormatFormFields()}
+                model={inputFormat}
+                onChange={p => this.updateSpecPreview(deepSet(spec, 'ioConfig.inputFormat', p))}
               />
               {this.renderApplyButtonBar()}
             </>
           )}
           {this.renderFlattenControls()}
-          {Boolean(sugestedFlattenFields && sugestedFlattenFields.length) && (
+          {sugestedFlattenFields && sugestedFlattenFields.length ? (
             <FormGroup>
               <Button
                 icon={IconNames.LIGHTBULB}
-                text="Auto add flatten specs"
+                text={`Auto add ${pluralIfNeeded(sugestedFlattenFields.length, 'flatten spec')}`}
                 onClick={() => {
                   this.updateSpec(
-                    deepSet(
-                      spec,
-                      'dataSchema.parser.parseSpec.flattenSpec.fields',
-                      sugestedFlattenFields,
-                    ),
+                    deepSet(spec, 'ioConfig.inputFormat.flattenSpec.fields', sugestedFlattenFields),
                   );
                 }}
               />
             </FormGroup>
+          ) : (
+            undefined
           )}
         </div>
         {this.renderNextBar({
@@ -1325,7 +1336,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           onNextStep: () => {
             if (!parserQueryState.data) return;
             let possibleTimestampSpec: TimestampSpec;
-            if (isIngestSegment(spec)) {
+            if (isDruidSource(spec)) {
               possibleTimestampSpec = {
                 column: '__time',
                 format: 'auto',
@@ -1337,7 +1348,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             if (possibleTimestampSpec) {
               const newSpec: IngestionSpec = deepSet(
                 spec,
-                'dataSchema.parser.parseSpec.timestampSpec',
+                'dataSchema.timestampSpec',
                 possibleTimestampSpec,
               );
               this.updateSpec(newSpec);
@@ -1357,8 +1368,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
   renderFlattenControls(): JSX.Element | undefined {
     const { spec, selectedFlattenField, selectedFlattenFieldIndex } = this.state;
-    const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || EMPTY_OBJECT;
-    if (!parseSpecHasFlatten(parseSpec)) return;
+    const inputFormat: InputFormat = deepGet(spec, 'ioConfig.inputFormat') || EMPTY_OBJECT;
+    if (!inputFormatCanFlatten(inputFormat)) return;
 
     const close = () => {
       this.setState({
@@ -1383,7 +1394,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 this.updateSpec(
                   deepSet(
                     spec,
-                    `dataSchema.parser.parseSpec.flattenSpec.fields.${selectedFlattenFieldIndex}`,
+                    `ioConfig.inputFormat.flattenSpec.fields.${selectedFlattenFieldIndex}`,
                     selectedFlattenField,
                   ),
                 );
@@ -1400,7 +1411,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                   this.updateSpec(
                     deepDelete(
                       spec,
-                      `dataSchema.parser.parseSpec.flattenSpec.fields.${selectedFlattenFieldIndex}`,
+                      `ioConfig.inputFormat.flattenSpec.fields.${selectedFlattenFieldIndex}`,
                     ),
                   );
                   close();
@@ -1436,23 +1447,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   // ==================================================================
 
   async queryForTimestamp(initRun = false) {
-    const { spec, sampleStrategy, cacheKey } = this.state;
-    const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
-    const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
-    const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
-    const timestampSpec =
-      deepGet(spec, 'dataSchema.parser.parseSpec.timestampSpec') || EMPTY_OBJECT;
+    const { spec, cacheRows } = this.state;
+    const inputFormatColumns: string[] =
+      deepGet(spec, 'ioConfig.inputFormat.columns') || EMPTY_ARRAY;
+    const timestampSpec = deepGet(spec, 'dataSchema.timestampSpec') || EMPTY_OBJECT;
 
-    let issue: string | undefined;
-    if (issueWithIoConfig(ioConfig)) {
-      issue = `IoConfig not ready, ${issueWithIoConfig(ioConfig)}`;
-    } else if (issueWithParser(parser)) {
-      issue = `Parser not ready, ${issueWithParser(parser)}`;
-    }
-
-    if (issue) {
+    if (!cacheRows) {
       this.setState({
-        timestampQueryState: initRun ? QueryState.INIT : new QueryState({ error: issue }),
+        timestampQueryState: initRun
+          ? QueryState.INIT
+          : new QueryState({ error: 'must complete parse step' }),
       });
       return;
     }
@@ -1463,7 +1467,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
     let sampleResponse: SampleResponse;
     try {
-      sampleResponse = await sampleForTimestamp(spec, sampleStrategy, cacheKey);
+      sampleResponse = await sampleForTimestamp(spec, cacheRows);
     } catch (e) {
       this.setState({
         timestampQueryState: new QueryState({ error: e.message }),
@@ -1472,13 +1476,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     }
 
     this.setState({
-      cacheKey: sampleResponse.cacheKey,
       timestampQueryState: new QueryState({
         data: {
           headerAndRows: headerAndRowsFromSampleResponse(
             sampleResponse,
             undefined,
-            ['__time'].concat(parserColumns),
+            ['__time'].concat(inputFormatColumns),
           ),
           timestampSpec,
         },
@@ -1488,8 +1491,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
   renderTimestampStep() {
     const { specPreview: spec, columnFilter, specialColumnsOnly, timestampQueryState } = this.state;
-    const timestampSpec: TimestampSpec =
-      deepGet(spec, 'dataSchema.parser.parseSpec.timestampSpec') || EMPTY_OBJECT;
+    const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec') || EMPTY_OBJECT;
     const timestampSpecFromColumn = isColumnTimestampSpec(timestampSpec);
 
     let mainFill: JSX.Element | string = '';
@@ -1558,9 +1560,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                     column: 'timestamp',
                     format: 'auto',
                   };
-                  this.updateSpecPreview(
-                    deepSet(spec, 'dataSchema.parser.parseSpec.timestampSpec', timestampSpec),
-                  );
+                  this.updateSpecPreview(deepSet(spec, 'dataSchema.timestampSpec', timestampSpec));
                 }}
               />
               <Button
@@ -1568,11 +1568,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 active={!timestampSpecFromColumn}
                 onClick={() => {
                   this.updateSpecPreview(
-                    deepSet(
-                      spec,
-                      'dataSchema.parser.parseSpec.timestampSpec',
-                      getEmptyTimestampSpec(),
-                    ),
+                    deepSet(spec, 'dataSchema.timestampSpec', getConstantTimestampSpec()),
                   );
                 }}
               />
@@ -1582,9 +1578,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             fields={getTimestampSpecFormFields(timestampSpec)}
             model={timestampSpec}
             onChange={timestampSpec => {
-              this.updateSpecPreview(
-                deepSet(spec, 'dataSchema.parser.parseSpec.timestampSpec', timestampSpec),
-              );
+              this.updateSpecPreview(deepSet(spec, 'dataSchema.timestampSpec', timestampSpec));
             }}
           />
           {this.renderApplyButtonBar()}
@@ -1598,29 +1592,21 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
   private onTimestampColumnSelect = (newTimestampSpec: TimestampSpec) => {
     const { specPreview } = this.state;
-    this.updateSpecPreview(
-      deepSet(specPreview, 'dataSchema.parser.parseSpec.timestampSpec', newTimestampSpec),
-    );
+    this.updateSpecPreview(deepSet(specPreview, 'dataSchema.timestampSpec', newTimestampSpec));
   };
 
   // ==================================================================
 
   async queryForTransform(initRun = false) {
-    const { spec, sampleStrategy, cacheKey } = this.state;
-    const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
-    const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
-    const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
+    const { spec, cacheRows } = this.state;
+    const inputFormatColumns: string[] =
+      deepGet(spec, 'ioConfig.inputFormat.columns') || EMPTY_ARRAY;
 
-    let issue: string | undefined;
-    if (issueWithIoConfig(ioConfig)) {
-      issue = `IoConfig not ready, ${issueWithIoConfig(ioConfig)}`;
-    } else if (issueWithParser(parser)) {
-      issue = `Parser not ready, ${issueWithParser(parser)}`;
-    }
-
-    if (issue) {
+    if (!cacheRows) {
       this.setState({
-        transformQueryState: initRun ? QueryState.INIT : new QueryState({ error: issue }),
+        transformQueryState: initRun
+          ? QueryState.INIT
+          : new QueryState({ error: 'must complete parse step' }),
       });
       return;
     }
@@ -1631,7 +1617,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
     let sampleResponse: SampleResponse;
     try {
-      sampleResponse = await sampleForTransform(spec, sampleStrategy, cacheKey);
+      sampleResponse = await sampleForTransform(spec, cacheRows);
     } catch (e) {
       this.setState({
         transformQueryState: new QueryState({ error: e.message }),
@@ -1640,12 +1626,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     }
 
     this.setState({
-      cacheKey: sampleResponse.cacheKey,
       transformQueryState: new QueryState({
         data: headerAndRowsFromSampleResponse(
           sampleResponse,
           undefined,
-          ['__time'].concat(parserColumns),
+          ['__time'].concat(inputFormatColumns),
         ),
       }),
     });
@@ -1741,7 +1726,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           disabled: !transformQueryState.data,
           onNextStep: () => {
             if (!transformQueryState.data) return;
-            if (!isIngestSegment(spec)) {
+            if (!isDruidSource(spec)) {
               this.updateSpec(
                 updateSchemaWithSample(spec, transformQueryState.data, 'specific', true),
               );
@@ -1832,21 +1817,15 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   // ==================================================================
 
   async queryForFilter(initRun = false) {
-    const { spec, sampleStrategy, cacheKey } = this.state;
-    const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
-    const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
-    const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
+    const { spec, cacheRows } = this.state;
+    const inputFormatColumns: string[] =
+      deepGet(spec, 'ioConfig.inputFormat.columns') || EMPTY_ARRAY;
 
-    let issue: string | undefined;
-    if (issueWithIoConfig(ioConfig)) {
-      issue = `IoConfig not ready, ${issueWithIoConfig(ioConfig)}`;
-    } else if (issueWithParser(parser)) {
-      issue = `Parser not ready, ${issueWithParser(parser)}`;
-    }
-
-    if (issue) {
+    if (!cacheRows) {
       this.setState({
-        filterQueryState: initRun ? QueryState.INIT : new QueryState({ error: issue }),
+        filterQueryState: initRun
+          ? QueryState.INIT
+          : new QueryState({ error: 'must complete parse step' }),
       });
       return;
     }
@@ -1857,7 +1836,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
     let sampleResponse: SampleResponse;
     try {
-      sampleResponse = await sampleForFilter(spec, sampleStrategy, cacheKey);
+      sampleResponse = await sampleForFilter(spec, cacheRows);
     } catch (e) {
       this.setState({
         filterQueryState: new QueryState({ error: e.message }),
@@ -1867,12 +1846,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
     if (sampleResponse.data.length) {
       this.setState({
-        cacheKey: sampleResponse.cacheKey,
         filterQueryState: new QueryState({
           data: headerAndRowsFromSampleResponse(
             sampleResponse,
             undefined,
-            ['__time'].concat(parserColumns),
+            ['__time'].concat(inputFormatColumns),
             true,
           ),
         }),
@@ -1884,7 +1862,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     let sampleResponseNoFilter: SampleResponse;
     try {
       const specNoFilter = deepSet(spec, 'dataSchema.transformSpec.filter', null);
-      sampleResponseNoFilter = await sampleForFilter(specNoFilter, sampleStrategy, cacheKey);
+      sampleResponseNoFilter = await sampleForFilter(specNoFilter, cacheRows);
     } catch (e) {
       this.setState({
         filterQueryState: new QueryState({ error: e.message }),
@@ -1895,12 +1873,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     const headerAndRowsNoFilter = headerAndRowsFromSampleResponse(
       sampleResponseNoFilter,
       undefined,
-      ['__time'].concat(parserColumns),
+      ['__time'].concat(inputFormatColumns),
       true,
     );
 
     this.setState({
-      cacheKey: sampleResponseNoFilter.cacheKey,
+      // cacheRows: sampleResponseNoFilter.cacheKey,
       filterQueryState: new QueryState({
         data: deepSet(headerAndRowsNoFilter, 'rows', []),
       }),
@@ -2113,24 +2091,18 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   // ==================================================================
 
   async queryForSchema(initRun = false) {
-    const { spec, sampleStrategy, cacheKey } = this.state;
-    const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
-    const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
-    const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
+    const { spec, cacheRows } = this.state;
+    const inputFormatColumns: string[] =
+      deepGet(spec, 'ioConfig.inputFormat.columns') || EMPTY_ARRAY;
     const metricsSpec: MetricSpec[] = deepGet(spec, 'dataSchema.metricsSpec') || EMPTY_ARRAY;
     const dimensionsSpec: DimensionsSpec =
-      deepGet(spec, 'dataSchema.parser.parseSpec.dimensionsSpec') || EMPTY_OBJECT;
+      deepGet(spec, 'dataSchema.dimensionsSpec') || EMPTY_OBJECT;
 
-    let issue: string | undefined;
-    if (issueWithIoConfig(ioConfig)) {
-      issue = `IoConfig not ready, ${issueWithIoConfig(ioConfig)}`;
-    } else if (issueWithParser(parser)) {
-      issue = `Parser not ready, ${issueWithParser(parser)}`;
-    }
-
-    if (issue) {
+    if (!cacheRows) {
       this.setState({
-        schemaQueryState: initRun ? QueryState.INIT : new QueryState({ error: issue }),
+        schemaQueryState: initRun
+          ? QueryState.INIT
+          : new QueryState({ error: 'must complete parse step' }),
       });
       return;
     }
@@ -2141,7 +2113,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
     let sampleResponse: SampleResponse;
     try {
-      sampleResponse = await sampleForSchema(spec, sampleStrategy, cacheKey);
+      sampleResponse = await sampleForSchema(spec, cacheRows);
     } catch (e) {
       this.setState({
         schemaQueryState: new QueryState({ error: e.message }),
@@ -2150,13 +2122,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     }
 
     this.setState({
-      cacheKey: sampleResponse.cacheKey,
       schemaQueryState: new QueryState({
         data: {
           headerAndRows: headerAndRowsFromSampleResponse(
             sampleResponse,
             undefined,
-            ['__time'].concat(parserColumns),
+            ['__time'].concat(inputFormatColumns),
           ),
           dimensionsSpec,
           metricsSpec,
@@ -2265,8 +2236,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 <AutoForm
                   fields={[
                     {
-                      name: 'dataSchema.parser.parseSpec.dimensionsSpec.dimensionExclusions',
-                      label: 'Exclusions',
+                      name: 'dataSchema.dimensionsSpec.dimensionExclusions',
+                      label: 'Dimension exclusions',
                       type: 'string-array',
                       info: (
                         <>
@@ -2368,13 +2339,13 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   };
 
   renderChangeRollupAction() {
-    const { newRollup, spec, sampleStrategy, cacheKey } = this.state;
-    if (typeof newRollup === 'undefined') return;
+    const { newRollup, spec, cacheRows } = this.state;
+    if (typeof newRollup === 'undefined' || !cacheRows) return;
 
     return (
       <AsyncActionDialog
         action={async () => {
-          const sampleResponse = await sampleForTransform(spec, sampleStrategy, cacheKey);
+          const sampleResponse = await sampleForTransform(spec, cacheRows);
           this.updateSpec(
             updateSchemaWithSample(
               spec,
@@ -2397,14 +2368,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   }
 
   renderChangeDimensionModeAction() {
-    const { newDimensionMode, spec, sampleStrategy, cacheKey } = this.state;
-    if (typeof newDimensionMode === 'undefined') return;
+    const { newDimensionMode, spec, cacheRows } = this.state;
+    if (typeof newDimensionMode === 'undefined' || !cacheRows) return;
     const autoDetect = newDimensionMode === 'auto-detect';
 
     return (
       <AsyncActionDialog
         action={async () => {
-          const sampleResponse = await sampleForTransform(spec, sampleStrategy, cacheKey);
+          const sampleResponse = await sampleForTransform(spec, cacheRows);
           this.updateSpec(
             updateSchemaWithSample(
               spec,
@@ -2443,13 +2414,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     };
 
     if (selectedDimensionSpec) {
-      const curDimensions =
-        deepGet(spec, `dataSchema.parser.parseSpec.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
+      const curDimensions = deepGet(spec, `dataSchema.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
 
       const convertToMetric = (type: string, prefix: string) => {
         const specWithoutDimension = deepDelete(
           spec,
-          `dataSchema.parser.parseSpec.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
+          `dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
         );
 
         const specWithMetric = deepSet(specWithoutDimension, `dataSchema.metricsSpec.[append]`, {
@@ -2518,7 +2488,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 this.updateSpec(
                   deepSet(
                     spec,
-                    `dataSchema.parser.parseSpec.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
+                    `dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
                     selectedDimensionSpec,
                   ),
                 );
@@ -2538,7 +2508,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                   this.updateSpec(
                     deepDelete(
                       spec,
-                      `dataSchema.parser.parseSpec.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
+                      `dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
                     ),
                   );
                   close();
@@ -2588,7 +2558,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
         const specWithDimension = deepSet(
           specWithoutMetric,
-          `dataSchema.parser.parseSpec.dimensionsSpec.dimensions.[append]`,
+          `dataSchema.dimensionsSpec.dimensions.[append]`,
           {
             type,
             name: selectedMetricSpec.fieldName,
@@ -2746,7 +2716,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         <div className="other">
           <H5>Secondary partitioning</H5>
           <AutoForm
-            fields={getPartitionRelatedTuningSpecFormFields(getSpecType(spec) || 'index')}
+            fields={getPartitionRelatedTuningSpecFormFields(getSpecType(spec) || 'index_parallel')}
             model={tuningConfig}
             onChange={t => this.updateSpec(deepSet(spec, 'tuningConfig', t))}
           />
@@ -2759,7 +2729,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/ingestion/index.html#partitioning`}
             />
           </Callout>
-          {this.renderParallelPickerIfNeeded()}
         </div>
         {this.renderNextBar({
           disabled:
@@ -2794,10 +2763,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               />
             ) : (
               <div>
-                {ioConfig.firehose
-                  ? `No specific tuning configs for firehose of type '${deepGet(
+                {ioConfig.inputSource
+                  ? `No specific tuning configs for inputSource of type '${deepGet(
                       ioConfig,
-                      'firehose.type',
+                      'inputSource.type',
                     )}'.`
                   : `No specific tuning configs.`}
               </div>
@@ -2826,7 +2795,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/ingestion/index.html#tuningconfig`}
             />
           </Callout>
-          {this.renderParallelPickerIfNeeded()}
         </div>
         {this.renderNextBar({
           disabled: invalidIoConfig(ioConfig),
@@ -2835,44 +2803,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     );
   }
 
-  renderParallelPickerIfNeeded(): JSX.Element | undefined {
-    const { spec } = this.state;
-    if (!hasParallelAbility(spec)) return;
-
-    return (
-      <FormGroup>
-        <Switch
-          large
-          checked={isParallel(spec)}
-          onChange={() => this.updateSpec(changeParallel(spec, !isParallel(spec)))}
-          labelElement={
-            <>
-              {'Parallel indexing '}
-              <Popover
-                content={
-                  <PopoverText>
-                    Druid currently has two types of native batch indexing tasks,{' '}
-                    <Code>index_parallel</Code> which runs tasks in parallel on multiple
-                    MiddleManager processes, and <Code>index</Code> which will run a single indexing
-                    task locally on a single MiddleManager.
-                  </PopoverText>
-                }
-                position="left-bottom"
-              >
-                <Icon icon={IconNames.INFO_SIGN} iconSize={16} />
-              </Popover>
-            </>
-          }
-        />
-      </FormGroup>
-    );
-  }
-
   // ==================================================================
 
   renderPublishStep() {
     const { spec } = this.state;
-    const parallel = isParallel(spec);
+    const parallel = deepGet(spec, 'tuningConfig.maxNumConcurrentSubTasks') > 1;
 
     return (
       <>
@@ -2962,7 +2897,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           <Callout className="intro">
             <p>Configure behavior of indexed data once it reaches Druid.</p>
           </Callout>
-          {this.renderParallelPickerIfNeeded()}
         </div>
         {this.renderNextBar({})}
       </>
@@ -3050,7 +2984,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   }
 
   private handleSubmit = async () => {
-    const { goToTask } = this.props;
+    const { goToIngestion } = this.props;
     const { spec, submitting } = this.state;
     if (submitting) return;
 
@@ -3080,7 +3014,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       });
 
       setTimeout(() => {
-        goToTask(taskResp.data.task);
+        goToIngestion(taskResp.data.task);
       }, 1000);
     } else {
       try {
@@ -3100,7 +3034,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       });
 
       setTimeout(() => {
-        goToTask(undefined); // Can we get the supervisor ID here?
+        goToIngestion(undefined); // Can we get the supervisor ID here?
       }, 1000);
     }
   };
diff --git a/web-console/src/views/load-data-view/parse-data-table/parse-data-table.spec.tsx b/web-console/src/views/load-data-view/parse-data-table/parse-data-table.spec.tsx
index 4ec2037a810..298e43bd6ca 100644
--- a/web-console/src/views/load-data-view/parse-data-table/parse-data-table.spec.tsx
+++ b/web-console/src/views/load-data-view/parse-data-table/parse-data-table.spec.tsx
@@ -27,7 +27,7 @@ describe('parse data table', () => {
       header: ['c1'],
       rows: [
         {
-          raw: `{"c1":"hello"}`,
+          input: { c1: 'hello' },
           parsed: { c1: 'hello' },
         },
       ],
diff --git a/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx b/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
index f51503d1054..6a6975d52f5 100644
--- a/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
+++ b/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
@@ -22,7 +22,7 @@ import ReactTable from 'react-table';
 
 import { TableCell } from '../../../components';
 import { TableCellUnparseable } from '../../../components/table-cell-unparseable/table-cell-unparseable';
-import { caseInsensitiveContains, filterMap, parseJson } from '../../../utils';
+import { caseInsensitiveContains, filterMap } from '../../../utils';
 import { FlattenField } from '../../../utils/ingestion-spec';
 import { HeaderAndRows, SampleEntry } from '../../../utils/sampler';
 
@@ -85,20 +85,16 @@ export const ParseDataTable = React.memo(function ParseDataTable(props: ParseDat
         };
       })}
       SubComponent={rowInfo => {
-        const { raw, error } = rowInfo.original;
-        const parsedJson: any = parseJson(raw);
+        const { input, error } = rowInfo.original;
+        const inputStr = JSON.stringify(input, null, 2);
 
-        if (!error && parsedJson && canFlatten) {
-          return (
-            <pre className="parse-detail">
-              {'Original row: ' + JSON.stringify(parsedJson, null, 2)}
-            </pre>
-          );
+        if (!error && input && canFlatten) {
+          return <pre className="parse-detail">{'Original row: ' + inputStr}</pre>;
         } else {
           return (
             <div className="parse-detail">
               {error && <div className="parse-error">{error}</div>}
-              <div>{'Original row: ' + rowInfo.original.raw}</div>
+              <div>{'Original row: ' + inputStr}</div>
             </div>
           );
         }
diff --git a/web-console/src/views/load-data-view/parse-time-table/parse-time-table.spec.tsx b/web-console/src/views/load-data-view/parse-time-table/parse-time-table.spec.tsx
index 89588455cf9..13b82fc247d 100644
--- a/web-console/src/views/load-data-view/parse-time-table/parse-time-table.spec.tsx
+++ b/web-console/src/views/load-data-view/parse-time-table/parse-time-table.spec.tsx
@@ -19,7 +19,7 @@
 import { render } from '@testing-library/react';
 import React from 'react';
 
-import { getEmptyTimestampSpec } from '../../../utils/ingestion-spec';
+import { getDummyTimestampSpec } from '../../../utils/ingestion-spec';
 
 import { ParseTimeTable } from './parse-time-table';
 
@@ -29,7 +29,7 @@ describe('parse time table', () => {
       header: ['c1'],
       rows: [
         {
-          raw: `{"c1":"hello"}`,
+          input: { c1: 'hello' },
           parsed: { c1: 'hello' },
         },
       ],
@@ -39,7 +39,7 @@ describe('parse time table', () => {
       <ParseTimeTable
         sampleBundle={{
           headerAndRows: sampleData,
-          timestampSpec: getEmptyTimestampSpec(),
+          timestampSpec: getDummyTimestampSpec(),
         }}
         columnFilter=""
         possibleTimestampColumnsOnly={false}
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
index 96956960ea6..2a4dad98c75 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
@@ -27,7 +27,7 @@ describe('schema table', () => {
       header: ['c1'],
       rows: [
         {
-          raw: `{"c1":"hello"}`,
+          input: { c1: 'hello' },
           parsed: { c1: 'hello' },
         },
       ],
diff --git a/web-console/src/views/load-data-view/transform-table/transform-table.spec.tsx b/web-console/src/views/load-data-view/transform-table/transform-table.spec.tsx
index 3dbade3835e..e801ad4ef21 100644
--- a/web-console/src/views/load-data-view/transform-table/transform-table.spec.tsx
+++ b/web-console/src/views/load-data-view/transform-table/transform-table.spec.tsx
@@ -27,7 +27,7 @@ describe('transform table', () => {
       header: ['c1'],
       rows: [
         {
-          raw: `{"c1":"hello"}`,
+          input: { c1: 'hello' },
           parsed: { c1: 'hello' },
         },
       ],
diff --git a/web-console/src/views/query-view/query-view.tsx b/web-console/src/views/query-view/query-view.tsx
index 45729071c2a..986b6da90b4 100644
--- a/web-console/src/views/query-view/query-view.tsx
+++ b/web-console/src/views/query-view/query-view.tsx
@@ -347,7 +347,7 @@ export class QueryView extends React.PureComponent<QueryViewProps, QueryViewStat
 
   prettyPrintJson(): void {
     this.setState(prevState => ({
-      queryString: Hjson.stringify(Hjson.parse(prevState.queryString)),
+      queryString: JSON.stringify(Hjson.parse(prevState.queryString), null, 2),
     }));
   }
 
diff --git a/web-console/src/views/query-view/run-button/run-button.tsx b/web-console/src/views/query-view/run-button/run-button.tsx
index e449fdd33ef..438d6f30e18 100644
--- a/web-console/src/views/query-view/run-button/run-button.tsx
+++ b/web-console/src/views/query-view/run-button/run-button.tsx
@@ -142,7 +142,7 @@ export class RunButton extends React.PureComponent<RunButtonProps> {
           />
         )}
         {runeMode && (
-          <MenuItem icon={IconNames.PRINT} text="Pretty print JSON" onClick={onPrettier} />
+          <MenuItem icon={IconNames.ALIGN_LEFT} text="Prettify JSON" onClick={onPrettier} />
         )}
       </Menu>
     );