From cb09b572e620aac83ed02c8e3a1832dcf68771fb Mon Sep 17 00:00:00 2001
From: zachjsh <zachjsh@gmail.com>
Date: Thu, 8 Aug 2024 21:21:03 -0400
Subject: [PATCH 01/99] Fix Druid  table schema resolution when table defined
 in catalog and has schema manager (#16869)

* SQL syntax error should target USER persona

* * revert change to queryHandler and related tests, based on review comments

* * add test

* Properly handle Druid schema blending with catalog definition and segment metadata

* * add javadocs
---
 .../apache/druid/sql/calcite/schema/DruidSchema.java | 12 +++++++++---
 .../druid/sql/calcite/schema/DruidSchemaManager.java | 11 ++++++++++-
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java
index 4e8431cd5c6..5a8ae5ba9a8 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java
@@ -22,6 +22,7 @@ package org.apache.druid.sql.calcite.schema;
 import org.apache.calcite.schema.Table;
 import org.apache.druid.sql.calcite.planner.CatalogResolver;
 import org.apache.druid.sql.calcite.table.DatasourceTable;
+import org.apache.druid.sql.calcite.table.DruidTable;
 
 import javax.inject.Inject;
 import java.util.Set;
@@ -56,11 +57,16 @@ public class DruidSchema extends AbstractTableSchema
   @Override
   public Table getTable(String name)
   {
-    if (druidSchemaManager != null) {
-      return druidSchemaManager.getTable(name);
-    } else {
+    DruidTable schemaMgrTable = null;
+    DruidTable catalogTable = catalogResolver.resolveDatasource(name, null);
+    if (catalogTable == null && druidSchemaManager != null) {
+      schemaMgrTable = druidSchemaManager.getTable(name, segmentMetadataCache);
+    }
+    if (schemaMgrTable == null) {
       DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentMetadataCache.getDatasource(name);
       return catalogResolver.resolveDatasource(name, dsMetadata);
+    } else {
+      return schemaMgrTable;
     }
   }
 
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchemaManager.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchemaManager.java
index c203ab18825..85808bb1af4 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchemaManager.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchemaManager.java
@@ -21,6 +21,7 @@ package org.apache.druid.sql.calcite.schema;
 
 import org.apache.druid.guice.annotations.ExtensionPoint;
 import org.apache.druid.guice.annotations.UnstableApi;
+import org.apache.druid.sql.calcite.planner.CatalogResolver;
 import org.apache.druid.sql.calcite.table.DruidTable;
 
 import java.util.Map;
@@ -29,7 +30,10 @@ import java.util.Set;
 /**
  * This interface provides a map of datasource names to {@link DruidTable}
  * objects, used by the {@link DruidSchema} class as the SQL planner's
- * view of Druid datasource schemas. If a non-default implementation is
+ * view of Druid datasource schemas. If a mapping is found for the
+ * datasource name in {@link CatalogResolver}, then it is preferred during
+ * resolution time, to the entry found in this manager. See
+ * {@link DruidSchema#getTable(String)}. If a non-default implementation is
  * provided, the segment metadata polling-based view of the Druid tables
  * will not be built in DruidSchema.
  */
@@ -56,6 +60,11 @@ public interface DruidSchemaManager
     return getTables().get(name);
   }
 
+  default DruidTable getTable(String name, BrokerSegmentMetadataCache segmentMetadataCache)
+  {
+    return getTables().get(name);
+  }
+
   default Set<String> getTableNames()
   {
     return getTables().keySet();

From 3d6cedb25fd81f2415b4f46b025f3624e8501550 Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Fri, 9 Aug 2024 11:39:53 +0530
Subject: [PATCH 02/99] Fix IndexOutOfBoundsException for MSQ window function
 queries with empty RAC (#16865)

* Fix IndexOutOfBoundsException for MSQ window function queries with empty RAC
---
 .../rowsandcols/ConcatRowsAndColumns.java     |  9 +++
 .../rowsandcols/ConcatRowsAndColumnsTest.java | 57 +++++++++++++++++++
 .../sql/calcite/DrillWindowQueryTest.java     |  7 +++
 .../empty_over_clause/single_empty_over_3.e   |  0
 .../empty_over_clause/single_empty_over_3.q   |  4 ++
 5 files changed, 77 insertions(+)
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/empty_over_clause/single_empty_over_3.e
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/empty_over_clause/single_empty_over_3.q

diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/ConcatRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/ConcatRowsAndColumns.java
index 523a982fbaf..c6ced60849d 100644
--- a/processing/src/main/java/org/apache/druid/query/rowsandcols/ConcatRowsAndColumns.java
+++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/ConcatRowsAndColumns.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.query.rowsandcols;
 
+import com.google.common.base.Preconditions;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.rowsandcols.column.Column;
 import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
@@ -30,6 +31,7 @@ import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -53,6 +55,7 @@ public class ConcatRowsAndColumns implements RowsAndColumns
       ArrayList<RowsAndColumns> racBuffer
   )
   {
+    Preconditions.checkNotNull(racBuffer, "racBuffer cannot be null");
     this.racBuffer = racBuffer;
 
     int numRows = 0;
@@ -76,6 +79,9 @@ public class ConcatRowsAndColumns implements RowsAndColumns
   @Override
   public Collection<String> getColumnNames()
   {
+    if (racBuffer.isEmpty()) {
+      return Collections.emptySet();
+    }
     return racBuffer.get(0).getColumnNames();
   }
 
@@ -92,6 +98,9 @@ public class ConcatRowsAndColumns implements RowsAndColumns
     if (columnCache.containsKey(name)) {
       return columnCache.get(name);
     } else {
+      if (racBuffer.isEmpty()) {
+        return null;
+      }
       final Column firstCol = racBuffer.get(0).findColumn(name);
       if (firstCol == null) {
         for (int i = 1; i < racBuffer.size(); ++i) {
diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/ConcatRowsAndColumnsTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/ConcatRowsAndColumnsTest.java
index b95f750632d..0bf0450114b 100644
--- a/processing/src/test/java/org/apache/druid/query/rowsandcols/ConcatRowsAndColumnsTest.java
+++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/ConcatRowsAndColumnsTest.java
@@ -19,7 +19,13 @@
 
 package org.apache.druid.query.rowsandcols;
 
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.query.rowsandcols.column.IntArrayColumn;
+import org.junit.Assert;
+import org.junit.Test;
+
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.function.Function;
 
 public class ConcatRowsAndColumnsTest extends RowsAndColumnsTestBase
@@ -42,4 +48,55 @@ public class ConcatRowsAndColumnsTest extends RowsAndColumnsTestBase
 
     return new ConcatRowsAndColumns(theRac);
   };
+
+  @Test
+  public void testConstructorWithNullRacBuffer()
+  {
+    final NullPointerException e = Assert.assertThrows(
+        NullPointerException.class,
+        () -> new ConcatRowsAndColumns(null)
+    );
+    Assert.assertEquals("racBuffer cannot be null", e.getMessage());
+  }
+
+  @Test
+  public void testFindColumn()
+  {
+    MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(
+        ImmutableMap.of(
+            "column1", new IntArrayColumn(new int[]{1, 2, 3, 4, 5, 6}),
+            "column2", new IntArrayColumn(new int[]{6, 5, 4, 3, 2, 1})
+        )
+    );
+    ConcatRowsAndColumns apply = MAKER.apply(rac);
+    Assert.assertEquals(1, apply.findColumn("column1").toAccessor().getInt(0));
+    Assert.assertEquals(6, apply.findColumn("column2").toAccessor().getInt(0));
+  }
+
+  @Test
+  public void testFindColumnWithEmptyRacBuffer()
+  {
+    ConcatRowsAndColumns concatRowsAndColumns = new ConcatRowsAndColumns(new ArrayList<>());
+    Assert.assertNull(concatRowsAndColumns.findColumn("columnName"));
+  }
+
+  @Test
+  public void testGetColumns()
+  {
+    MapOfColumnsRowsAndColumns rac = MapOfColumnsRowsAndColumns.fromMap(
+        ImmutableMap.of(
+            "column1", new IntArrayColumn(new int[]{0, 0, 0, 1, 1, 2, 4, 4, 4}),
+            "column2", new IntArrayColumn(new int[]{3, 54, 21, 1, 5, 54, 2, 3, 92})
+        )
+    );
+    ConcatRowsAndColumns apply = MAKER.apply(rac);
+    Assert.assertEquals(Arrays.asList("column1", "column2"), new ArrayList<>(apply.getColumnNames()));
+  }
+
+  @Test
+  public void testGetColumnsWithEmptyRacBuffer()
+  {
+    ConcatRowsAndColumns concatRowsAndColumns = new ConcatRowsAndColumns(new ArrayList<>());
+    Assert.assertTrue(concatRowsAndColumns.getColumnNames().isEmpty());
+  }
 }
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
index 4a2f0945087..9bf56d97d38 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
@@ -7622,6 +7622,13 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
     windowQueryTest();
   }
 
+  @DrillTest("druid_queries/empty_over_clause/single_empty_over_3")
+  @Test
+  public void test_empty_over_single_empty_over_3()
+  {
+    windowQueryTest();
+  }
+
   @DrillTest("druid_queries/empty_over_clause/multiple_empty_over_1")
   @Test
   public void test_empty_over_multiple_empty_over_1()
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/empty_over_clause/single_empty_over_3.e b/sql/src/test/resources/drill/window/queries/druid_queries/empty_over_clause/single_empty_over_3.e
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/empty_over_clause/single_empty_over_3.q b/sql/src/test/resources/drill/window/queries/druid_queries/empty_over_clause/single_empty_over_3.q
new file mode 100644
index 00000000000..ac8c5e930d3
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/empty_over_clause/single_empty_over_3.q
@@ -0,0 +1,4 @@
+select countryName, row_number() over () as c1
+from wikipedia
+where countryName in ('non-existent-country')
+group by countryName, cityName, channel

From a7dd436a32a91473b6f3ce5d599a03b603b00f6c Mon Sep 17 00:00:00 2001
From: Adithya Chakilam <35785271+adithyachakilam@users.noreply.github.com>
Date: Fri, 9 Aug 2024 04:12:48 -0500
Subject: [PATCH 03/99] Check if supervisor could be idle on startup (#16844)

Fixes #13936

In cases where a supervisor is idle and the overlord is restarted for some reason, the supervisor would
start spinning tasks again. In clusters where there are many low throughput streams, this would spike
the task count unnecessarily.

This commit compares the latest stream offset with the ones in metadata during the startup of supervisor
and sets it to idle state if they match.
---
 .../kafka/supervisor/KafkaSupervisorTest.java |   3 -
 .../supervisor/SeekableStreamSupervisor.java  |  15 ++-
 .../SeekableStreamSupervisorStateTest.java    | 104 +++++++++++++++++-
 .../indexer/AbstractStreamIndexingTest.java   |  20 ++++
 .../supervisor/SupervisorStateManager.java    |   5 +-
 5 files changed, 136 insertions(+), 11 deletions(-)

diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
index b18c1749125..86275d10e31 100644
--- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
+++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
@@ -408,13 +408,10 @@ public class KafkaSupervisorTest extends EasyMockSupport
     autoscaler.start();
     supervisor.runInternal();
     Thread.sleep(1000);
-    supervisor.runInternal();
     verifyAll();
 
     int taskCountAfterScale = supervisor.getIoConfig().getTaskCount();
     Assert.assertEquals(2, taskCountAfterScale);
-    Assert.assertEquals(SupervisorStateManager.BasicState.IDLE, supervisor.getState());
-
 
     KafkaIndexTask task = captured.getValue();
     Assert.assertEquals(KafkaSupervisorTest.dataSchema, task.getDataSchema());
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
index a99c782557b..81d8871b8e8 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
@@ -3640,12 +3640,21 @@ public abstract class SeekableStreamSupervisor<PartitionIdType, SequenceOffsetTy
       return;
     }
 
-    Map<PartitionIdType, SequenceOffsetType> latestSequencesFromStream = getLatestSequencesFromStream();
     final long nowTime = DateTimes.nowUtc().getMillis();
+    // if it is the first run and there is no lag observed when compared to the offsets from metadata storage stay idle
+    if (!stateManager.isAtLeastOneSuccessfulRun()) {
+      // Set previous sequences to the current offsets in metadata store
+      previousSequencesFromStream.clear();
+      previousSequencesFromStream.putAll(getOffsetsFromMetadataStorage());
+
+      // Force update partition lag since the reporting thread might not have run yet
+      updatePartitionLagFromStream();
+    }
+
+    Map<PartitionIdType, SequenceOffsetType> latestSequencesFromStream = getLatestSequencesFromStream();
     final boolean idle;
     final long idleTime;
-    if (lastActiveTimeMillis > 0
-        && previousSequencesFromStream.equals(latestSequencesFromStream)
+    if (previousSequencesFromStream.equals(latestSequencesFromStream)
         && computeTotalLag() == 0) {
       idleTime = nowTime - lastActiveTimeMillis;
       idle = true;
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java
index cb395acf66b..daf85ac39c9 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java
@@ -709,8 +709,8 @@ public class SeekableStreamSupervisorStateTest extends EasyMockSupport
 
     replayAll();
 
-    SeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor();
-
+    TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor();
+    supervisor.setStreamOffsets(ImmutableMap.of("0", "10"));
     supervisor.start();
 
     Assert.assertTrue(supervisor.stateManager.isHealthy());
@@ -757,6 +757,93 @@ public class SeekableStreamSupervisorStateTest extends EasyMockSupport
     verifyAll();
   }
 
+  @Test
+  public void testIdleOnStartUpAndTurnsToRunningAfterLagUpdates()
+  {
+    Map<String, String> initialOffsets = ImmutableMap.of("0", "10");
+    Map<String, String> laterOffsets = ImmutableMap.of("0", "20");
+
+    EasyMock.reset(indexerMetadataStorageCoordinator);
+    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(
+        new TestSeekableStreamDataSourceMetadata(
+            new SeekableStreamEndSequenceNumbers<>(
+                STREAM,
+                initialOffsets
+            )
+        )
+    ).anyTimes();
+    EasyMock.reset(spec);
+    EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes();
+    EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes();
+    EasyMock.expect(spec.getContextValue("tags")).andReturn("").anyTimes();
+    EasyMock.expect(spec.getIoConfig()).andReturn(new SeekableStreamSupervisorIOConfig(
+        "stream",
+        new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of()), ImmutableMap.of(), false, false, false),
+        1,
+        1,
+        new Period("PT1H"),
+        new Period("PT1S"),
+        new Period("PT30S"),
+        false,
+        new Period("PT30M"),
+        null,
+        null,
+        null,
+        null,
+        new IdleConfig(true, 200L),
+        null
+    )
+    {
+    }).anyTimes();
+    EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes();
+    EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes();
+    EasyMock.expect(spec.getMonitorSchedulerConfig()).andReturn(new DruidMonitorSchedulerConfig()
+    {
+      @Override
+      public Duration getEmissionDuration()
+      {
+        return new Period("PT1S").toStandardDuration();
+      }
+    }).anyTimes();
+    EasyMock.expect(spec.getType()).andReturn("test").anyTimes();
+    EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes();
+    EasyMock.expect(recordSupplier.getPartitionIds(STREAM)).andReturn(ImmutableSet.of(SHARD_ID)).anyTimes();
+    EasyMock.expect(recordSupplier.isOffsetAvailable(EasyMock.anyObject(), EasyMock.anyObject()))
+            .andReturn(true)
+            .anyTimes();
+    EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
+    EasyMock.expect(taskQueue.add(EasyMock.anyObject())).andReturn(true).anyTimes();
+    replayAll();
+
+    TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor();
+
+    supervisor.start();
+
+    Assert.assertTrue(supervisor.stateManager.isHealthy());
+    Assert.assertEquals(BasicState.PENDING, supervisor.stateManager.getSupervisorState());
+    Assert.assertEquals(BasicState.PENDING, supervisor.stateManager.getSupervisorState().getBasicState());
+    Assert.assertTrue(supervisor.stateManager.getExceptionEvents().isEmpty());
+    Assert.assertFalse(supervisor.stateManager.isAtLeastOneSuccessfulRun());
+
+    supervisor.setStreamOffsets(initialOffsets);
+    supervisor.runInternal();
+
+    Assert.assertTrue(supervisor.stateManager.isHealthy());
+    Assert.assertEquals(BasicState.IDLE, supervisor.stateManager.getSupervisorState());
+    Assert.assertEquals(BasicState.IDLE, supervisor.stateManager.getSupervisorState().getBasicState());
+    Assert.assertTrue(supervisor.stateManager.getExceptionEvents().isEmpty());
+    Assert.assertTrue(supervisor.stateManager.isAtLeastOneSuccessfulRun());
+
+    supervisor.setStreamOffsets(laterOffsets);
+    supervisor.runInternal();
+
+    Assert.assertTrue(supervisor.stateManager.isHealthy());
+    Assert.assertEquals(BasicState.RUNNING, supervisor.stateManager.getSupervisorState());
+    Assert.assertEquals(BasicState.RUNNING, supervisor.stateManager.getSupervisorState().getBasicState());
+    Assert.assertTrue(supervisor.stateManager.getExceptionEvents().isEmpty());
+    Assert.assertTrue(supervisor.stateManager.isAtLeastOneSuccessfulRun());
+  }
+
   @Test
   public void testCreatingTasksFailRecoveryFail()
   {
@@ -2872,6 +2959,8 @@ public class SeekableStreamSupervisorStateTest extends EasyMockSupport
 
   private class TestSeekableStreamSupervisor extends BaseTestSeekableStreamSupervisor
   {
+    Map<String, String> streamOffsets = new HashMap<>();
+
     @Override
     protected void scheduleReporting(ScheduledExecutorService reportingExec)
     {
@@ -2883,6 +2972,17 @@ public class SeekableStreamSupervisorStateTest extends EasyMockSupport
     {
       return new LagStats(0, 0, 0);
     }
+
+    @Override
+    protected Map<String, String> getLatestSequencesFromStream()
+    {
+      return streamOffsets;
+    }
+
+    public void setStreamOffsets(Map<String, String> streamOffsets)
+    {
+      this.streamOffsets = streamOffsets;
+    }
   }
 
   private class TestEmittingTestSeekableStreamSupervisor extends BaseTestSeekableStreamSupervisor
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
index e20c2ea2061..8cc8388ba47 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
@@ -561,6 +561,26 @@ public abstract class AbstractStreamIndexingTest extends AbstractIndexerTest
           "wait for no more creation of indexing tasks"
       );
 
+      indexer.shutdownSupervisor(generatedTestConfig.getSupervisorId());
+      indexer.submitSupervisor(taskSpec);
+
+      ITRetryUtil.retryUntil(
+          () -> SupervisorStateManager.BasicState.IDLE.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())),
+          true,
+          10000,
+          30,
+          "Waiting for supervisor to be idle"
+      );
+      ITRetryUtil.retryUntil(
+          () -> indexer.getRunningTasks()
+                       .stream()
+                       .noneMatch(taskResponseObject -> taskResponseObject.getId().contains(dataSource)),
+          true,
+          1000,
+          10,
+          "wait for no more creation of indexing tasks"
+      );
+
       // Start generating remainning half of the data
       numWritten += streamGenerator.run(
           generatedTestConfig.getStreamName(),
diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorStateManager.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorStateManager.java
index 1ea36229c38..88049f18b08 100644
--- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorStateManager.java
+++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorStateManager.java
@@ -155,11 +155,10 @@ public class SupervisorStateManager
       return;
     }
 
-    // if we're trying to switch to a healthy steady state (i.e. RUNNING or SUSPENDED) or IDLE state but haven't had a successful run
+    // if we're trying to switch to a healthy steady state (i.e. RUNNING or SUSPENDED) but haven't had a successful run
     // yet, refuse to switch and prefer the more specific states used for first run (CONNECTING_TO_STREAM,
     // DISCOVERING_INITIAL_TASKS, CREATING_TASKS, etc.)
-    if ((healthySteadyState.equals(proposedState) || BasicState.IDLE.equals(proposedState))
-        && !atLeastOneSuccessfulRun) {
+    if (healthySteadyState.equals(proposedState) && !atLeastOneSuccessfulRun) {
       return;
     }
 

From 483a03f26c7221f10493d86715985a58e61af967 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Fri, 9 Aug 2024 14:46:59 -0700
Subject: [PATCH 04/99] Web console: Server context defaults (#16868)

* add server defaults

* null is NULL

* r to d

* add test

* typo
---
 licenses.yaml                                 |   2 +-
 web-console/console-config.js                 |   3 +-
 web-console/package-lock.json                 |  14 +-
 web-console/package.json                      |   2 +-
 .../src/components/header-bar/header-bar.tsx  |   3 +-
 web-console/src/console-application.tsx       |  63 +++--
 .../query-context/query-context.tsx           | 251 +++---------------
 web-console/src/entry.tsx                     |  14 +-
 .../helpers/execution/sql-task-execution.ts   |  16 +-
 web-console/src/utils/general.tsx             |   6 +
 web-console/src/utils/values-query.spec.tsx   |   4 +-
 web-console/src/utils/values-query.tsx        |  21 +-
 .../sql-data-loader-view.tsx                  |  12 +-
 .../max-tasks-button.spec.tsx                 |   8 +-
 .../max-tasks-button/max-tasks-button.tsx     |  47 ++--
 .../workbench-view/query-tab/query-tab.tsx    |  32 ++-
 .../workbench-view/run-panel/run-panel.tsx    | 210 +++++++++------
 .../views/workbench-view/workbench-view.tsx   |  46 +++-
 18 files changed, 343 insertions(+), 411 deletions(-)

diff --git a/licenses.yaml b/licenses.yaml
index dcdac7bd187..0646c7131fd 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -5085,7 +5085,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Imply Data
-version: 0.22.20
+version: 0.22.21
 
 ---
 
diff --git a/web-console/console-config.js b/web-console/console-config.js
index 10bdddb611a..25d99e7c650 100644
--- a/web-console/console-config.js
+++ b/web-console/console-config.js
@@ -17,6 +17,5 @@
  */
 
 window.consoleConfig = {
-  exampleManifestsUrl: 'https://druid.apache.org/data/example-manifests-v2.tsv',
-  /* future configs may go here */
+  /* configs go here */
 };
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index 412f728d56d..e9319969b69 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -14,7 +14,7 @@
         "@blueprintjs/datetime2": "^2.3.7",
         "@blueprintjs/icons": "^5.10.0",
         "@blueprintjs/select": "^5.2.1",
-        "@druid-toolkit/query": "^0.22.20",
+        "@druid-toolkit/query": "^0.22.21",
         "@druid-toolkit/visuals-core": "^0.3.3",
         "@druid-toolkit/visuals-react": "^0.3.3",
         "@fontsource/open-sans": "^5.0.28",
@@ -989,9 +989,9 @@
       }
     },
     "node_modules/@druid-toolkit/query": {
-      "version": "0.22.20",
-      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.20.tgz",
-      "integrity": "sha512-GmmSd27y7zLVTjgTBQy+XoGeSSGhSDNmwyiwWtSua7I5LX8XqHV7Chi8HIH25YQoVgTK1pLK4RS8eRXxthRAzg==",
+      "version": "0.22.21",
+      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.21.tgz",
+      "integrity": "sha512-4k0NGO2Ay90naSO8nyivPPvvhz73D/OkCo6So3frmPDLFfw5CYKSvAhy4RadtnLMZPwsnlVREjAmqbvBsHqgjQ==",
       "dependencies": {
         "tslib": "^2.5.2"
       }
@@ -19093,9 +19093,9 @@
       "dev": true
     },
     "@druid-toolkit/query": {
-      "version": "0.22.20",
-      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.20.tgz",
-      "integrity": "sha512-GmmSd27y7zLVTjgTBQy+XoGeSSGhSDNmwyiwWtSua7I5LX8XqHV7Chi8HIH25YQoVgTK1pLK4RS8eRXxthRAzg==",
+      "version": "0.22.21",
+      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.21.tgz",
+      "integrity": "sha512-4k0NGO2Ay90naSO8nyivPPvvhz73D/OkCo6So3frmPDLFfw5CYKSvAhy4RadtnLMZPwsnlVREjAmqbvBsHqgjQ==",
       "requires": {
         "tslib": "^2.5.2"
       }
diff --git a/web-console/package.json b/web-console/package.json
index 0c9370f8808..d55bb79d609 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -68,7 +68,7 @@
     "@blueprintjs/datetime2": "^2.3.7",
     "@blueprintjs/icons": "^5.10.0",
     "@blueprintjs/select": "^5.2.1",
-    "@druid-toolkit/query": "^0.22.20",
+    "@druid-toolkit/query": "^0.22.21",
     "@druid-toolkit/visuals-core": "^0.3.3",
     "@druid-toolkit/visuals-react": "^0.3.3",
     "@fontsource/open-sans": "^5.0.28",
diff --git a/web-console/src/components/header-bar/header-bar.tsx b/web-console/src/components/header-bar/header-bar.tsx
index e1b97cf4e13..aed66798299 100644
--- a/web-console/src/components/header-bar/header-bar.tsx
+++ b/web-console/src/components/header-bar/header-bar.tsx
@@ -59,7 +59,6 @@ import './header-bar.scss';
 const capabilitiesOverride = localStorageGetJson(LocalStorageKeys.CAPABILITIES_OVERRIDE);
 
 export type HeaderActiveTab =
-  | null
   | 'data-loader'
   | 'streaming-data-loader'
   | 'classic-batch-data-loader'
@@ -93,7 +92,7 @@ const DruidLogo = React.memo(function DruidLogo() {
 });
 
 export interface HeaderBarProps {
-  active: HeaderActiveTab;
+  active: HeaderActiveTab | null;
   capabilities: Capabilities;
   onUnrestrict(capabilities: Capabilities): void;
 }
diff --git a/web-console/src/console-application.tsx b/web-console/src/console-application.tsx
index 0d097729cbf..36a0b8aa392 100644
--- a/web-console/src/console-application.tsx
+++ b/web-console/src/console-application.tsx
@@ -28,7 +28,7 @@ import type { Filter } from 'react-table';
 
 import type { HeaderActiveTab } from './components';
 import { HeaderBar, Loader } from './components';
-import type { DruidEngine, QueryWithContext } from './druid-models';
+import type { DruidEngine, QueryContext, QueryWithContext } from './druid-models';
 import { Capabilities, maybeGetClusterCapacity } from './helpers';
 import { stringToTableFilters, tableFiltersToString } from './react-table';
 import { AppToaster } from './singletons';
@@ -51,22 +51,32 @@ import './console-application.scss';
 
 type FiltersRouteMatch = RouteComponentProps<{ filters?: string }>;
 
-function changeHashWithFilter(slug: string, filters: Filter[]) {
+function changeTabWithFilter(tab: HeaderActiveTab, filters: Filter[]) {
   const filterString = tableFiltersToString(filters);
-  location.hash = slug + (filterString ? `/${filterString}` : '');
+  location.hash = tab + (filterString ? `/${filterString}` : '');
 }
 
-function viewFilterChange(slug: string) {
-  return (filters: Filter[]) => changeHashWithFilter(slug, filters);
+function viewFilterChange(tab: HeaderActiveTab) {
+  return (filters: Filter[]) => changeTabWithFilter(tab, filters);
 }
 
-function pathWithFilter(slug: string) {
-  return [`/${slug}/:filters`, `/${slug}`];
+function pathWithFilter(tab: HeaderActiveTab) {
+  return [`/${tab}/:filters`, `/${tab}`];
+}
+
+function switchTab(tab: HeaderActiveTab) {
+  location.hash = tab;
+}
+
+function switchToWorkbenchTab(tabId: string) {
+  location.hash = `workbench/${tabId}`;
 }
 
 export interface ConsoleApplicationProps {
-  defaultQueryContext?: Record<string, any>;
-  mandatoryQueryContext?: Record<string, any>;
+  baseQueryContext?: QueryContext;
+  defaultQueryContext?: QueryContext;
+  mandatoryQueryContext?: QueryContext;
+  serverQueryContext?: QueryContext;
 }
 
 export interface ConsoleApplicationState {
@@ -158,22 +168,22 @@ export class ConsoleApplication extends React.PureComponent<
 
   private readonly goToStreamingDataLoader = (supervisorId?: string) => {
     if (supervisorId) this.supervisorId = supervisorId;
-    location.hash = 'streaming-data-loader';
+    switchTab('streaming-data-loader');
     this.resetInitialsWithDelay();
   };
 
   private readonly goToClassicBatchDataLoader = (taskId?: string) => {
     if (taskId) this.taskId = taskId;
-    location.hash = 'classic-batch-data-loader';
+    switchTab('classic-batch-data-loader');
     this.resetInitialsWithDelay();
   };
 
   private readonly goToDatasources = (datasource: string) => {
-    changeHashWithFilter('datasources', [{ id: 'datasource', value: `=${datasource}` }]);
+    changeTabWithFilter('datasources', [{ id: 'datasource', value: `=${datasource}` }]);
   };
 
   private readonly goToSegments = (datasource: string, onlyUnavailable = false) => {
-    changeHashWithFilter(
+    changeTabWithFilter(
       'segments',
       compact([
         { id: 'datasource', value: `=${datasource}` },
@@ -183,19 +193,19 @@ export class ConsoleApplication extends React.PureComponent<
   };
 
   private readonly goToSupervisor = (supervisorId: string) => {
-    changeHashWithFilter('supervisors', [{ id: 'supervisor_id', value: `=${supervisorId}` }]);
+    changeTabWithFilter('supervisors', [{ id: 'supervisor_id', value: `=${supervisorId}` }]);
   };
 
   private readonly goToTasksWithTaskId = (taskId: string) => {
-    changeHashWithFilter('tasks', [{ id: 'task_id', value: `=${taskId}` }]);
+    changeTabWithFilter('tasks', [{ id: 'task_id', value: `=${taskId}` }]);
   };
 
   private readonly goToTasksWithTaskGroupId = (taskGroupId: string) => {
-    changeHashWithFilter('tasks', [{ id: 'group_id', value: `=${taskGroupId}` }]);
+    changeTabWithFilter('tasks', [{ id: 'group_id', value: `=${taskGroupId}` }]);
   };
 
   private readonly goToTasksWithDatasource = (datasource: string, type?: string) => {
-    changeHashWithFilter(
+    changeTabWithFilter(
       'tasks',
       compact([
         { id: 'datasource', value: `=${datasource}` },
@@ -206,24 +216,24 @@ export class ConsoleApplication extends React.PureComponent<
 
   private readonly openSupervisorSubmit = () => {
     this.openSupervisorDialog = true;
-    location.hash = 'supervisors';
+    switchTab('supervisors');
     this.resetInitialsWithDelay();
   };
 
   private readonly openTaskSubmit = () => {
     this.openTaskDialog = true;
-    location.hash = 'tasks';
+    switchTab('tasks');
     this.resetInitialsWithDelay();
   };
 
   private readonly goToQuery = (queryWithContext: QueryWithContext) => {
     this.queryWithContext = queryWithContext;
-    location.hash = 'workbench';
+    switchTab('workbench');
     this.resetInitialsWithDelay();
   };
 
   private readonly wrapInViewContainer = (
-    active: HeaderActiveTab,
+    active: HeaderActiveTab | null,
     el: JSX.Element,
     classType: 'normal' | 'narrow-pad' | 'thin' | 'thinner' = 'normal',
   ) => {
@@ -293,7 +303,8 @@ export class ConsoleApplication extends React.PureComponent<
   };
 
   private readonly wrappedWorkbenchView = (p: RouteComponentProps<{ tabId?: string }>) => {
-    const { defaultQueryContext, mandatoryQueryContext } = this.props;
+    const { defaultQueryContext, mandatoryQueryContext, baseQueryContext, serverQueryContext } =
+      this.props;
     const { capabilities } = this.state;
 
     const queryEngines: DruidEngine[] = ['native'];
@@ -309,12 +320,12 @@ export class ConsoleApplication extends React.PureComponent<
       <WorkbenchView
         capabilities={capabilities}
         tabId={p.match.params.tabId}
-        onTabChange={newTabId => {
-          location.hash = `workbench/${newTabId}`;
-        }}
+        onTabChange={switchToWorkbenchTab}
         initQueryWithContext={this.queryWithContext}
         defaultQueryContext={defaultQueryContext}
         mandatoryQueryContext={mandatoryQueryContext}
+        baseQueryContext={baseQueryContext}
+        serverQueryContext={serverQueryContext}
         queryEngines={queryEngines}
         allowExplain
         goToTask={this.goToTasksWithTaskId}
@@ -325,6 +336,7 @@ export class ConsoleApplication extends React.PureComponent<
   };
 
   private readonly wrappedSqlDataLoaderView = () => {
+    const { serverQueryContext } = this.props;
     const { capabilities } = this.state;
     return this.wrapInViewContainer(
       'sql-data-loader',
@@ -334,6 +346,7 @@ export class ConsoleApplication extends React.PureComponent<
         goToTask={this.goToTasksWithTaskId}
         goToTaskGroup={this.goToTasksWithTaskGroupId}
         getClusterCapacity={maybeGetClusterCapacity}
+        serverQueryContext={serverQueryContext}
       />,
     );
   };
diff --git a/web-console/src/druid-models/query-context/query-context.tsx b/web-console/src/druid-models/query-context/query-context.tsx
index 17e8204e949..a25d268d845 100644
--- a/web-console/src/druid-models/query-context/query-context.tsx
+++ b/web-console/src/druid-models/query-context/query-context.tsx
@@ -16,9 +16,10 @@
  * limitations under the License.
  */
 
-import { deepDelete, deepSet } from '../../utils';
-
+export type SelectDestination = 'taskReport' | 'durableStorage';
 export type ArrayIngestMode = 'array' | 'mvd';
+export type TaskAssignment = 'auto' | 'max';
+export type SqlJoinAlgorithm = 'broadcast' | 'sortMerge';
 
 export interface QueryContext {
   useCache?: boolean;
@@ -30,15 +31,38 @@ export interface QueryContext {
   // Multi-stage query
   maxNumTasks?: number;
   finalizeAggregations?: boolean;
-  selectDestination?: string;
+  selectDestination?: SelectDestination;
   durableShuffleStorage?: boolean;
   maxParseExceptions?: number;
   groupByEnableMultiValueUnnesting?: boolean;
   arrayIngestMode?: ArrayIngestMode;
+  taskAssignment?: TaskAssignment;
+  sqlJoinAlgorithm?: SqlJoinAlgorithm;
+  failOnEmptyInsert?: boolean;
+  waitUntilSegmentsLoad?: boolean;
 
   [key: string]: any;
 }
 
+export const DEFAULT_SERVER_QUERY_CONTEXT: QueryContext = {
+  useCache: true,
+  populateCache: true,
+  useApproximateCountDistinct: true,
+  useApproximateTopN: true,
+  sqlTimeZone: 'Etc/UTC',
+
+  // Multi-stage query
+  finalizeAggregations: true,
+  selectDestination: 'taskReport',
+  durableShuffleStorage: false,
+  maxParseExceptions: 0,
+  groupByEnableMultiValueUnnesting: true,
+  taskAssignment: 'max',
+  sqlJoinAlgorithm: 'broadcast',
+  failOnEmptyInsert: false,
+  waitUntilSegmentsLoad: false,
+};
+
 export interface QueryWithContext {
   queryString: string;
   queryContext?: QueryContext;
@@ -49,221 +73,10 @@ export function isEmptyContext(context: QueryContext | undefined): boolean {
   return !context || Object.keys(context).length === 0;
 }
 
-// -----------------------------
-
-export function getUseCache(context: QueryContext): boolean {
-  const { useCache } = context;
-  return typeof useCache === 'boolean' ? useCache : true;
-}
-
-export function changeUseCache(context: QueryContext, useCache: boolean): QueryContext {
-  let newContext = context;
-  if (useCache) {
-    newContext = deepDelete(newContext, 'useCache');
-    newContext = deepDelete(newContext, 'populateCache');
-  } else {
-    newContext = deepSet(newContext, 'useCache', false);
-    newContext = deepSet(newContext, 'populateCache', false);
-  }
-  return newContext;
-}
-
-// -----------------------------
-
-export function getUseApproximateCountDistinct(context: QueryContext): boolean {
-  const { useApproximateCountDistinct } = context;
-  return typeof useApproximateCountDistinct === 'boolean' ? useApproximateCountDistinct : true;
-}
-
-export function changeUseApproximateCountDistinct(
+export function getQueryContextKey(
+  key: keyof QueryContext,
   context: QueryContext,
-  useApproximateCountDistinct: boolean,
-): QueryContext {
-  if (useApproximateCountDistinct) {
-    return deepDelete(context, 'useApproximateCountDistinct');
-  } else {
-    return deepSet(context, 'useApproximateCountDistinct', false);
-  }
-}
-
-// -----------------------------
-
-export function getUseApproximateTopN(context: QueryContext): boolean {
-  const { useApproximateTopN } = context;
-  return typeof useApproximateTopN === 'boolean' ? useApproximateTopN : true;
-}
-
-export function changeUseApproximateTopN(
-  context: QueryContext,
-  useApproximateTopN: boolean,
-): QueryContext {
-  if (useApproximateTopN) {
-    return deepDelete(context, 'useApproximateTopN');
-  } else {
-    return deepSet(context, 'useApproximateTopN', false);
-  }
-}
-
-// sqlTimeZone
-
-export function getTimezone(context: QueryContext): string | undefined {
-  return context.sqlTimeZone;
-}
-
-export function changeTimezone(context: QueryContext, timezone: string | undefined): QueryContext {
-  if (timezone) {
-    return deepSet(context, 'sqlTimeZone', timezone);
-  } else {
-    return deepDelete(context, 'sqlTimeZone');
-  }
-}
-
-// maxNumTasks
-
-export function getMaxNumTasks(context: QueryContext): number | undefined {
-  return context.maxNumTasks;
-}
-
-export function changeMaxNumTasks(
-  context: QueryContext,
-  maxNumTasks: number | undefined,
-): QueryContext {
-  return typeof maxNumTasks === 'number'
-    ? deepSet(context, 'maxNumTasks', maxNumTasks)
-    : deepDelete(context, 'maxNumTasks');
-}
-
-// taskAssignment
-
-export function getTaskAssigment(context: QueryContext): string {
-  const { taskAssignment } = context;
-  return taskAssignment ?? 'max';
-}
-
-export function changeTaskAssigment(
-  context: QueryContext,
-  taskAssignment: string | undefined,
-): QueryContext {
-  return typeof taskAssignment === 'string'
-    ? deepSet(context, 'taskAssignment', taskAssignment)
-    : deepDelete(context, 'taskAssignment');
-}
-
-// failOnEmptyInsert
-
-export function getFailOnEmptyInsert(context: QueryContext): boolean | undefined {
-  const { failOnEmptyInsert } = context;
-  return typeof failOnEmptyInsert === 'boolean' ? failOnEmptyInsert : undefined;
-}
-
-export function changeFailOnEmptyInsert(
-  context: QueryContext,
-  failOnEmptyInsert: boolean | undefined,
-): QueryContext {
-  return typeof failOnEmptyInsert === 'boolean'
-    ? deepSet(context, 'failOnEmptyInsert', failOnEmptyInsert)
-    : deepDelete(context, 'failOnEmptyInsert');
-}
-
-// finalizeAggregations
-
-export function getFinalizeAggregations(context: QueryContext): boolean | undefined {
-  const { finalizeAggregations } = context;
-  return typeof finalizeAggregations === 'boolean' ? finalizeAggregations : undefined;
-}
-
-export function changeFinalizeAggregations(
-  context: QueryContext,
-  finalizeAggregations: boolean | undefined,
-): QueryContext {
-  return typeof finalizeAggregations === 'boolean'
-    ? deepSet(context, 'finalizeAggregations', finalizeAggregations)
-    : deepDelete(context, 'finalizeAggregations');
-}
-
-// waitUntilSegmentsLoad
-
-export function getWaitUntilSegmentsLoad(context: QueryContext): boolean | undefined {
-  const { waitUntilSegmentsLoad } = context;
-  return typeof waitUntilSegmentsLoad === 'boolean' ? waitUntilSegmentsLoad : undefined;
-}
-
-export function changeWaitUntilSegmentsLoad(
-  context: QueryContext,
-  waitUntilSegmentsLoad: boolean | undefined,
-): QueryContext {
-  return typeof waitUntilSegmentsLoad === 'boolean'
-    ? deepSet(context, 'waitUntilSegmentsLoad', waitUntilSegmentsLoad)
-    : deepDelete(context, 'waitUntilSegmentsLoad');
-}
-
-// groupByEnableMultiValueUnnesting
-
-export function getGroupByEnableMultiValueUnnesting(context: QueryContext): boolean | undefined {
-  const { groupByEnableMultiValueUnnesting } = context;
-  return typeof groupByEnableMultiValueUnnesting === 'boolean'
-    ? groupByEnableMultiValueUnnesting
-    : undefined;
-}
-
-export function changeGroupByEnableMultiValueUnnesting(
-  context: QueryContext,
-  groupByEnableMultiValueUnnesting: boolean | undefined,
-): QueryContext {
-  return typeof groupByEnableMultiValueUnnesting === 'boolean'
-    ? deepSet(context, 'groupByEnableMultiValueUnnesting', groupByEnableMultiValueUnnesting)
-    : deepDelete(context, 'groupByEnableMultiValueUnnesting');
-}
-
-// durableShuffleStorage
-
-export function getDurableShuffleStorage(context: QueryContext): boolean {
-  const { durableShuffleStorage } = context;
-  return Boolean(durableShuffleStorage);
-}
-
-export function changeDurableShuffleStorage(
-  context: QueryContext,
-  durableShuffleStorage: boolean,
-): QueryContext {
-  if (durableShuffleStorage) {
-    return deepSet(context, 'durableShuffleStorage', true);
-  } else {
-    return deepDelete(context, 'durableShuffleStorage');
-  }
-}
-
-// maxParseExceptions
-
-export function getMaxParseExceptions(context: QueryContext): number {
-  const { maxParseExceptions } = context;
-  return Number(maxParseExceptions) || 0;
-}
-
-export function changeMaxParseExceptions(
-  context: QueryContext,
-  maxParseExceptions: number,
-): QueryContext {
-  if (maxParseExceptions !== 0) {
-    return deepSet(context, 'maxParseExceptions', maxParseExceptions);
-  } else {
-    return deepDelete(context, 'maxParseExceptions');
-  }
-}
-
-// arrayIngestMode
-
-export function getArrayIngestMode(context: QueryContext): ArrayIngestMode | undefined {
-  return context.arrayIngestMode;
-}
-
-export function changeArrayIngestMode(
-  context: QueryContext,
-  arrayIngestMode: ArrayIngestMode | undefined,
-): QueryContext {
-  if (arrayIngestMode) {
-    return deepSet(context, 'arrayIngestMode', arrayIngestMode);
-  } else {
-    return deepDelete(context, 'arrayIngestMode');
-  }
+  defaultContext: QueryContext,
+): any {
+  return typeof context[key] !== 'undefined' ? context[key] : defaultContext[key];
 }
diff --git a/web-console/src/entry.tsx b/web-console/src/entry.tsx
index 25518ecdeb9..0e698a3f84b 100644
--- a/web-console/src/entry.tsx
+++ b/web-console/src/entry.tsx
@@ -28,6 +28,7 @@ import { createRoot } from 'react-dom/client';
 import { bootstrapJsonParse } from './bootstrap/json-parser';
 import { bootstrapReactTable } from './bootstrap/react-table-defaults';
 import { ConsoleApplication } from './console-application';
+import type { QueryContext } from './druid-models';
 import type { Links } from './links';
 import { setLinkOverrides } from './links';
 import { Api, UrlBaser } from './singletons';
@@ -55,11 +56,16 @@ interface ConsoleConfig {
   // A set of custom headers name/value to set on every AJAX request
   customHeaders?: Record<string, string>;
 
-  // The query context to set if the user does not have one saved in local storage, defaults to {}
-  defaultQueryContext?: Record<string, any>;
+  baseQueryContext?: QueryContext;
+
+  // The query context to set one new query tabs
+  defaultQueryContext?: QueryContext;
 
   // Extra context properties that will be added to all query requests
-  mandatoryQueryContext?: Record<string, any>;
+  mandatoryQueryContext?: QueryContext;
+
+  // The default context that is set by the server
+  serverQueryContext?: QueryContext;
 
   // Allow for link overriding to different docs
   linkOverrides?: Links;
@@ -104,8 +110,10 @@ QueryRunner.defaultQueryExecutor = (payload, isSql, cancelToken) => {
 createRoot(container).render(
   <OverlaysProvider>
     <ConsoleApplication
+      baseQueryContext={consoleConfig.baseQueryContext}
       defaultQueryContext={consoleConfig.defaultQueryContext}
       mandatoryQueryContext={consoleConfig.mandatoryQueryContext}
+      serverQueryContext={consoleConfig.serverQueryContext}
     />
   </OverlaysProvider>,
 );
diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts
index 0fa9b090959..f4dd45a2cb9 100644
--- a/web-console/src/helpers/execution/sql-task-execution.ts
+++ b/web-console/src/helpers/execution/sql-task-execution.ts
@@ -36,6 +36,7 @@ function ensureExecutionModeIsSet(context: QueryContext | undefined): QueryConte
 export interface SubmitTaskQueryOptions {
   query: string | Record<string, any>;
   context?: QueryContext;
+  baseQueryContext?: QueryContext;
   prefixLines?: number;
   cancelToken?: CancelToken;
   preserveOnTermination?: boolean;
@@ -45,7 +46,15 @@ export interface SubmitTaskQueryOptions {
 export async function submitTaskQuery(
   options: SubmitTaskQueryOptions,
 ): Promise<Execution | IntermediateQueryState<Execution>> {
-  const { query, context, prefixLines, cancelToken, preserveOnTermination, onSubmitted } = options;
+  const {
+    query,
+    context,
+    baseQueryContext,
+    prefixLines,
+    cancelToken,
+    preserveOnTermination,
+    onSubmitted,
+  } = options;
 
   let sqlQuery: string;
   let jsonQuery: Record<string, any>;
@@ -53,7 +62,7 @@ export async function submitTaskQuery(
     sqlQuery = query;
     jsonQuery = {
       query: sqlQuery,
-      context: ensureExecutionModeIsSet(context),
+      context: ensureExecutionModeIsSet({ ...baseQueryContext, ...context }),
       resultFormat: 'array',
       header: true,
       typesHeader: true,
@@ -65,6 +74,7 @@ export async function submitTaskQuery(
     jsonQuery = {
       ...query,
       context: ensureExecutionModeIsSet({
+        ...baseQueryContext,
         ...query.context,
         ...context,
       }),
@@ -96,7 +106,7 @@ export async function submitTaskQuery(
     );
   }
 
-  const execution = Execution.fromAsyncStatus(sqlAsyncStatus, sqlQuery, context);
+  const execution = Execution.fromAsyncStatus(sqlAsyncStatus, sqlQuery, jsonQuery.context);
 
   if (onSubmitted) {
     onSubmitted(execution.id);
diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx
index a3256c3ab11..7698d3c3af8 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -389,6 +389,12 @@ export function assemble<T>(...xs: (T | undefined | false | null | '')[]): T[] {
   return compact(xs);
 }
 
+export function removeUndefinedValues<T extends Record<string, any>>(obj: T): Partial<T> {
+  return Object.fromEntries(
+    Object.entries(obj).filter(([_, value]) => value !== undefined),
+  ) as Partial<T>;
+}
+
 export function moveToEnd<T>(
   xs: T[],
   predicate: (value: T, index: number, array: T[]) => unknown,
diff --git a/web-console/src/utils/values-query.spec.tsx b/web-console/src/utils/values-query.spec.tsx
index 99884f382c1..7bc093bc3e8 100644
--- a/web-console/src/utils/values-query.spec.tsx
+++ b/web-console/src/utils/values-query.spec.tsx
@@ -45,6 +45,7 @@ describe('queryResultToValuesQuery', () => {
           [2, 3],
           null,
         ],
+        [null, null, null, null, null, null, null],
       ],
       false,
       true,
@@ -64,7 +65,8 @@ describe('queryResultToValuesQuery', () => {
       FROM (
         VALUES
         ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"sys","swap/free":1223334,"swap/max":3223334}', 'es<#>es-419', '1', NULL),
-        ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"query","time":1223,"bytes":2434234}', 'en<#>es<#>es-419', '2<#>3', NULL)
+        ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"query","time":1223,"bytes":2434234}', 'en<#>es<#>es-419', '2<#>3', NULL),
+        (NULL, NULL, NULL, NULL, NULL, NULL, NULL)
       ) AS "t" ("c1", "c2", "c3", "c4", "c5", "c6", "c7")
     `);
   });
diff --git a/web-console/src/utils/values-query.tsx b/web-console/src/utils/values-query.tsx
index 2f1a5f699ca..1b5e62b44c2 100644
--- a/web-console/src/utils/values-query.tsx
+++ b/web-console/src/utils/values-query.tsx
@@ -65,28 +65,29 @@ export function queryResultToValuesQuery(sample: QueryResult): SqlQuery {
       expression: SqlValues.create(
         rows.map(row =>
           SqlRecord.create(
-            row.map((r, i) => {
+            row.map((d, i) => {
+              if (d == null) return L.NULL;
               const column = header[i];
               const { nativeType } = column;
               const sqlType = getEffectiveSqlType(column);
               if (nativeType === 'COMPLEX<json>') {
-                return L(isJsonString(r) ? r : JSONBig.stringify(r));
+                return L(isJsonString(d) ? d : JSONBig.stringify(d));
               } else if (String(sqlType).endsWith(' ARRAY')) {
-                return L(r.join(SAMPLE_ARRAY_SEPARATOR));
+                return L(d.join(SAMPLE_ARRAY_SEPARATOR));
               } else if (
                 sqlType === 'OTHER' &&
                 String(nativeType).startsWith('COMPLEX<') &&
-                typeof r === 'string' &&
-                r.startsWith('"') &&
-                r.endsWith('"')
+                typeof d === 'string' &&
+                d.startsWith('"') &&
+                d.endsWith('"')
               ) {
-                // r is a JSON encoded base64 string
-                return L(r.slice(1, -1));
-              } else if (typeof r === 'object') {
+                // d is a JSON encoded base64 string
+                return L(d.slice(1, -1));
+              } else if (typeof d === 'object') {
                 // Cleanup array if it happens to get here, it shouldn't.
                 return L.NULL;
               } else {
-                return L(r);
+                return L(d);
               }
             }),
           ),
diff --git a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
index 6cc00957b8d..2e10734a170 100644
--- a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
+++ b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
@@ -30,6 +30,7 @@ import type {
   QueryWithContext,
 } from '../../druid-models';
 import {
+  DEFAULT_SERVER_QUERY_CONTEXT,
   Execution,
   externalConfigToIngestQueryPattern,
   ingestQueryPatternToQuery,
@@ -65,12 +66,20 @@ export interface SqlDataLoaderViewProps {
   goToTask(taskId: string): void;
   goToTaskGroup(taskGroupId: string): void;
   getClusterCapacity: (() => Promise<CapacityInfo | undefined>) | undefined;
+  serverQueryContext?: QueryContext;
 }
 
 export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
   props: SqlDataLoaderViewProps,
 ) {
-  const { capabilities, goToQuery, goToTask, goToTaskGroup, getClusterCapacity } = props;
+  const {
+    capabilities,
+    goToQuery,
+    goToTask,
+    goToTaskGroup,
+    getClusterCapacity,
+    serverQueryContext = DEFAULT_SERVER_QUERY_CONTEXT,
+  } = props;
   const [alertElement, setAlertElement] = useState<JSX.Element | undefined>();
   const [externalConfigStep, setExternalConfigStep] = useState<Partial<ExternalConfig>>({});
   const [content, setContent] = useLocalStorageState<LoaderContent | undefined>(
@@ -187,6 +196,7 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
               clusterCapacity={capabilities.getMaxTaskSlots()}
               queryContext={content.queryContext || {}}
               changeQueryContext={queryContext => setContent({ ...content, queryContext })}
+              defaultQueryContext={serverQueryContext}
               minimal
             />
           }
diff --git a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.spec.tsx b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.spec.tsx
index 5954c1f3f9b..1ae864dee06 100644
--- a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.spec.tsx
+++ b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.spec.tsx
@@ -18,6 +18,7 @@
 
 import React from 'react';
 
+import { DEFAULT_SERVER_QUERY_CONTEXT } from '../../../druid-models';
 import { shallow } from '../../../utils/shallow-renderer';
 
 import { MaxTasksButton } from './max-tasks-button';
@@ -25,7 +26,12 @@ import { MaxTasksButton } from './max-tasks-button';
 describe('MaxTasksButton', () => {
   it('matches snapshot', () => {
     const comp = shallow(
-      <MaxTasksButton clusterCapacity={6} queryContext={{}} changeQueryContext={() => {}} />,
+      <MaxTasksButton
+        clusterCapacity={6}
+        queryContext={{}}
+        changeQueryContext={() => {}}
+        defaultQueryContext={DEFAULT_SERVER_QUERY_CONTEXT}
+      />,
     );
 
     expect(comp).toMatchSnapshot();
diff --git a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
index ea239bc3a26..c84f9f00e37 100644
--- a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
+++ b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
@@ -19,37 +19,38 @@
 import type { ButtonProps } from '@blueprintjs/core';
 import { Button, Menu, MenuDivider, MenuItem, Popover, Position } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
+import type { JSX } from 'react';
 import React, { useState } from 'react';
 
 import { NumericInputDialog } from '../../../dialogs';
-import type { QueryContext } from '../../../druid-models';
-import {
-  changeMaxNumTasks,
-  changeTaskAssigment,
-  getMaxNumTasks,
-  getTaskAssigment,
-} from '../../../druid-models';
-import { formatInteger, tickIcon } from '../../../utils';
+import type { QueryContext, TaskAssignment } from '../../../druid-models';
+import { getQueryContextKey } from '../../../druid-models';
+import { deleteKeys, formatInteger, tickIcon } from '../../../utils';
 
 const MAX_NUM_TASK_OPTIONS = [2, 3, 4, 5, 7, 9, 11, 17, 33, 65, 129];
-const TASK_ASSIGNMENT_OPTIONS = ['max', 'auto'];
+const TASK_ASSIGNMENT_OPTIONS: TaskAssignment[] = ['max', 'auto'];
 
 const TASK_ASSIGNMENT_DESCRIPTION: Record<string, string> = {
   max: 'Use as many tasks as possible, up to the maximum.',
   auto: `Use as few tasks as possible without exceeding 512 MiB or 10,000 files per task, unless exceeding these limits is necessary to stay within 'maxNumTasks'. When calculating the size of files, the weighted size is used, which considers the file format and compression format used if any. When file sizes cannot be determined through directory listing (for example: http), behaves the same as 'max'.`,
 };
 
-const DEFAULT_MAX_NUM_LABEL_FN = (maxNum: number) => {
+const DEFAULT_MAX_NUM_TASKS_LABEL_FN = (maxNum: number) => {
   if (maxNum === 2) return { text: formatInteger(maxNum), label: '(1 controller + 1 worker)' };
   return { text: formatInteger(maxNum), label: `(1 controller + max ${maxNum - 1} workers)` };
 };
 
+const DEFAULT_FULL_CLUSTER_CAPACITY_LABEL_FN = (clusterCapacity: number) =>
+  `${formatInteger(clusterCapacity)} (full cluster capacity)`;
+
 export interface MaxTasksButtonProps extends Omit<ButtonProps, 'text' | 'rightIcon'> {
   clusterCapacity: number | undefined;
   queryContext: QueryContext;
   changeQueryContext(queryContext: QueryContext): void;
+  defaultQueryContext: QueryContext;
   menuHeader?: JSX.Element;
-  maxNumLabelFn?: (maxNum: number) => { text: string; label?: string };
+  maxTasksLabelFn?: (maxNum: number) => { text: string; label?: string };
+  fullClusterCapacityLabelFn?: (clusterCapacity: number) => string;
 }
 
 export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps) {
@@ -57,19 +58,19 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
     clusterCapacity,
     queryContext,
     changeQueryContext,
+    defaultQueryContext,
     menuHeader,
-    maxNumLabelFn = DEFAULT_MAX_NUM_LABEL_FN,
+    maxTasksLabelFn = DEFAULT_MAX_NUM_TASKS_LABEL_FN,
+    fullClusterCapacityLabelFn = DEFAULT_FULL_CLUSTER_CAPACITY_LABEL_FN,
     ...rest
   } = props;
   const [customMaxNumTasksDialogOpen, setCustomMaxNumTasksDialogOpen] = useState(false);
 
-  const maxNumTasks = getMaxNumTasks(queryContext);
-  const taskAssigment = getTaskAssigment(queryContext);
+  const maxNumTasks = queryContext.maxNumTasks;
+  const taskAssigment = getQueryContextKey('taskAssignment', queryContext, defaultQueryContext);
 
   const fullClusterCapacity =
-    typeof clusterCapacity === 'number'
-      ? `${formatInteger(clusterCapacity)} (full cluster capacity)`
-      : undefined;
+    typeof clusterCapacity === 'number' ? fullClusterCapacityLabelFn(clusterCapacity) : undefined;
 
   const shownMaxNumTaskOptions = clusterCapacity
     ? MAX_NUM_TASK_OPTIONS.filter(_ => _ <= clusterCapacity)
@@ -88,11 +89,11 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
               <MenuItem
                 icon={tickIcon(typeof maxNumTasks === 'undefined')}
                 text={fullClusterCapacity}
-                onClick={() => changeQueryContext(changeMaxNumTasks(queryContext, undefined))}
+                onClick={() => changeQueryContext(deleteKeys(queryContext, ['maxNumTasks']))}
               />
             )}
             {shownMaxNumTaskOptions.map(m => {
-              const { text, label } = maxNumLabelFn(m);
+              const { text, label } = maxTasksLabelFn(m);
 
               return (
                 <MenuItem
@@ -100,7 +101,7 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
                   icon={tickIcon(m === maxNumTasks)}
                   text={text}
                   label={label}
-                  onClick={() => changeQueryContext(changeMaxNumTasks(queryContext, m))}
+                  onClick={() => changeQueryContext({ ...queryContext, maxNumTasks: m })}
                 />
               );
             })}
@@ -124,7 +125,7 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
                   }
                   shouldDismissPopover={false}
                   multiline
-                  onClick={() => changeQueryContext(changeTaskAssigment(queryContext, t))}
+                  onClick={() => changeQueryContext({ ...queryContext, taskAssignment: t })}
                 />
               ))}
             </MenuItem>
@@ -158,8 +159,8 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
           minValue={2}
           integer
           initValue={maxNumTasks || 2}
-          onSubmit={p => {
-            changeQueryContext(changeMaxNumTasks(queryContext, p));
+          onSubmit={maxNumTasks => {
+            changeQueryContext({ ...queryContext, maxNumTasks });
           }}
           onClose={() => setCustomMaxNumTasksDialogOpen(false)}
         />
diff --git a/web-console/src/views/workbench-view/query-tab/query-tab.tsx b/web-console/src/views/workbench-view/query-tab/query-tab.tsx
index cf863a14387..acdfa67fad2 100644
--- a/web-console/src/views/workbench-view/query-tab/query-tab.tsx
+++ b/web-console/src/views/workbench-view/query-tab/query-tab.tsx
@@ -21,14 +21,14 @@ import { IconNames } from '@blueprintjs/icons';
 import type { QueryResult } from '@druid-toolkit/query';
 import { QueryRunner, SqlQuery } from '@druid-toolkit/query';
 import axios from 'axios';
-import type { ComponentProps, JSX } from 'react';
+import type { JSX } from 'react';
 import React, { useCallback, useEffect, useRef, useState } from 'react';
 import SplitterLayout from 'react-splitter-layout';
 import { useStore } from 'zustand';
 
 import { Loader, QueryErrorPane } from '../../../components';
 import type { CapacityInfo, DruidEngine, LastExecution, QueryContext } from '../../../druid-models';
-import { Execution, WorkbenchQuery } from '../../../druid-models';
+import { DEFAULT_SERVER_QUERY_CONTEXT, Execution, WorkbenchQuery } from '../../../druid-models';
 import {
   executionBackgroundStatusCheck,
   reattachTaskExecution,
@@ -60,6 +60,7 @@ import { FlexibleQueryInput } from '../flexible-query-input/flexible-query-input
 import { IngestSuccessPane } from '../ingest-success-pane/ingest-success-pane';
 import { metadataStateStore } from '../metadata-state-store';
 import { ResultTablePane } from '../result-table-pane/result-table-pane';
+import type { RunPanelProps } from '../run-panel/run-panel';
 import { RunPanel } from '../run-panel/run-panel';
 import { workStateStore } from '../work-state-store';
 
@@ -69,10 +70,16 @@ const queryRunner = new QueryRunner({
   inflateDateStrategy: 'none',
 });
 
-export interface QueryTabProps {
+export interface QueryTabProps
+  extends Pick<
+    RunPanelProps,
+    'maxTasksMenuHeader' | 'enginesLabelFn' | 'maxTasksLabelFn' | 'fullClusterCapacityLabelFn'
+  > {
   query: WorkbenchQuery;
   id: string;
   mandatoryQueryContext: QueryContext | undefined;
+  baseQueryContext: QueryContext | undefined;
+  serverQueryContext: QueryContext;
   columnMetadata: readonly ColumnMetadata[] | undefined;
   onQueryChange(newQuery: WorkbenchQuery): void;
   onQueryTab(newQuery: WorkbenchQuery, tabName?: string): void;
@@ -82,9 +89,6 @@ export interface QueryTabProps {
   clusterCapacity: number | undefined;
   goToTask(taskId: string): void;
   getClusterCapacity: (() => Promise<CapacityInfo | undefined>) | undefined;
-  maxTaskMenuHeader?: JSX.Element;
-  enginesLabelFn?: ComponentProps<typeof RunPanel>['enginesLabelFn'];
-  maxTaskLabelFn?: ComponentProps<typeof RunPanel>['maxTaskLabelFn'];
 }
 
 export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
@@ -93,6 +97,8 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
     id,
     columnMetadata,
     mandatoryQueryContext,
+    baseQueryContext,
+    serverQueryContext = DEFAULT_SERVER_QUERY_CONTEXT,
     onQueryChange,
     onQueryTab,
     onDetails,
@@ -101,9 +107,10 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
     clusterCapacity,
     goToTask,
     getClusterCapacity,
-    maxTaskMenuHeader,
+    maxTasksMenuHeader,
     enginesLabelFn,
-    maxTaskLabelFn,
+    maxTasksLabelFn,
+    fullClusterCapacityLabelFn,
   } = props;
   const [alertElement, setAlertElement] = useState<JSX.Element | undefined>();
 
@@ -196,6 +203,8 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
           case 'sql-msq-task':
             return await submitTaskQuery({
               query,
+              context: mandatoryQueryContext,
+              baseQueryContext,
               prefixLines,
               cancelToken,
               preserveOnTermination: true,
@@ -227,6 +236,7 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
               const resultPromise = queryRunner.runQuery({
                 query,
                 extraQueryContext: mandatoryQueryContext,
+                defaultQueryContext: baseQueryContext,
                 cancelToken: new axios.CancelToken(cancelFn => {
                   nativeQueryCancelFnRef.current = cancelFn;
                 }),
@@ -404,10 +414,12 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
               running={executionState.loading}
               queryEngines={queryEngines}
               clusterCapacity={clusterCapacity}
+              defaultQueryContext={{ ...serverQueryContext, ...baseQueryContext }}
               moreMenu={runMoreMenu}
-              maxTaskMenuHeader={maxTaskMenuHeader}
+              maxTasksMenuHeader={maxTasksMenuHeader}
               enginesLabelFn={enginesLabelFn}
-              maxTaskLabelFn={maxTaskLabelFn}
+              maxTasksLabelFn={maxTasksLabelFn}
+              fullClusterCapacityLabelFn={fullClusterCapacityLabelFn}
             />
             {executionState.isLoading() && (
               <ExecutionTimerPanel
diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
index f8235379612..9ed135da354 100644
--- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx
+++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
@@ -30,7 +30,7 @@ import {
   useHotkeys,
 } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
-import type { ComponentProps, JSX } from 'react';
+import type { JSX } from 'react';
 import React, { useCallback, useMemo, useState } from 'react';
 
 import { MenuCheckbox, MenuTristate } from '../../../components';
@@ -41,35 +41,14 @@ import type {
   DruidEngine,
   IndexSpec,
   QueryContext,
+  SelectDestination,
+  SqlJoinAlgorithm,
   WorkbenchQuery,
 } from '../../../druid-models';
-import {
-  changeArrayIngestMode,
-  changeDurableShuffleStorage,
-  changeFailOnEmptyInsert,
-  changeFinalizeAggregations,
-  changeGroupByEnableMultiValueUnnesting,
-  changeMaxParseExceptions,
-  changeTimezone,
-  changeUseApproximateCountDistinct,
-  changeUseApproximateTopN,
-  changeUseCache,
-  changeWaitUntilSegmentsLoad,
-  getArrayIngestMode,
-  getDurableShuffleStorage,
-  getFailOnEmptyInsert,
-  getFinalizeAggregations,
-  getGroupByEnableMultiValueUnnesting,
-  getMaxParseExceptions,
-  getTimezone,
-  getUseApproximateCountDistinct,
-  getUseApproximateTopN,
-  getUseCache,
-  getWaitUntilSegmentsLoad,
-  summarizeIndexSpec,
-} from '../../../druid-models';
+import { getQueryContextKey, summarizeIndexSpec } from '../../../druid-models';
 import { getLink } from '../../../links';
-import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils';
+import { deepGet, pluralIfNeeded, removeUndefinedValues, tickIcon } from '../../../utils';
+import type { MaxTasksButtonProps } from '../max-tasks-button/max-tasks-button';
 import { MaxTasksButton } from '../max-tasks-button/max-tasks-button';
 import { QueryParametersDialog } from '../query-parameters-dialog/query-parameters-dialog';
 
@@ -119,7 +98,8 @@ const DEFAULT_ENGINES_LABEL_FN = (engine: DruidEngine | undefined) => {
   };
 };
 
-export interface RunPanelProps {
+export interface RunPanelProps
+  extends Pick<MaxTasksButtonProps, 'maxTasksLabelFn' | 'fullClusterCapacityLabelFn'> {
   query: WorkbenchQuery;
   onQueryChange(query: WorkbenchQuery): void;
   running: boolean;
@@ -127,10 +107,10 @@ export interface RunPanelProps {
   onRun(preview: boolean): void | Promise<void>;
   queryEngines: DruidEngine[];
   clusterCapacity: number | undefined;
+  defaultQueryContext: QueryContext;
   moreMenu?: JSX.Element;
-  maxTaskMenuHeader?: JSX.Element;
+  maxTasksMenuHeader?: JSX.Element;
   enginesLabelFn?: (engine: DruidEngine | undefined) => { text: string; label?: string };
-  maxTaskLabelFn?: ComponentProps<typeof MaxTasksButton>['maxNumLabelFn'];
 }
 
 export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
@@ -143,9 +123,11 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
     small,
     queryEngines,
     clusterCapacity,
-    maxTaskMenuHeader,
-    maxTaskLabelFn,
+    defaultQueryContext,
+    maxTasksMenuHeader,
     enginesLabelFn = DEFAULT_ENGINES_LABEL_FN,
+    maxTasksLabelFn,
+    fullClusterCapacityLabelFn,
   } = props;
   const [editContextDialogOpen, setEditContextDialogOpen] = useState(false);
   const [editParametersDialogOpen, setEditParametersDialogOpen] = useState(false);
@@ -158,20 +140,52 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
   const numContextKeys = Object.keys(queryContext).length;
   const queryParameters = query.queryParameters;
 
-  const arrayIngestMode = getArrayIngestMode(queryContext);
-  const maxParseExceptions = getMaxParseExceptions(queryContext);
-  const failOnEmptyInsert = getFailOnEmptyInsert(queryContext);
-  const finalizeAggregations = getFinalizeAggregations(queryContext);
-  const waitUntilSegmentsLoad = getWaitUntilSegmentsLoad(queryContext);
-  const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext);
-  const sqlJoinAlgorithm = queryContext.sqlJoinAlgorithm ?? 'broadcast';
-  const selectDestination = queryContext.selectDestination ?? 'taskReport';
-  const durableShuffleStorage = getDurableShuffleStorage(queryContext);
+  // Extract the context parts that have UI
+  const sqlTimeZone = queryContext.sqlTimeZone;
+
+  const useCache = getQueryContextKey('useCache', queryContext, defaultQueryContext);
+  const useApproximateTopN = getQueryContextKey(
+    'useApproximateTopN',
+    queryContext,
+    defaultQueryContext,
+  );
+  const useApproximateCountDistinct = getQueryContextKey(
+    'useApproximateCountDistinct',
+    queryContext,
+    defaultQueryContext,
+  );
+
+  const arrayIngestMode = queryContext.arrayIngestMode;
+  const maxParseExceptions = getQueryContextKey(
+    'maxParseExceptions',
+    queryContext,
+    defaultQueryContext,
+  );
+  const failOnEmptyInsert = getQueryContextKey(
+    'failOnEmptyInsert',
+    queryContext,
+    defaultQueryContext,
+  );
+  const finalizeAggregations = queryContext.finalizeAggregations;
+  const waitUntilSegmentsLoad = queryContext.waitUntilSegmentsLoad;
+  const groupByEnableMultiValueUnnesting = queryContext.groupByEnableMultiValueUnnesting;
+  const sqlJoinAlgorithm = getQueryContextKey(
+    'sqlJoinAlgorithm',
+    queryContext,
+    defaultQueryContext,
+  );
+  const selectDestination = getQueryContextKey(
+    'selectDestination',
+    queryContext,
+    defaultQueryContext,
+  );
+  const durableShuffleStorage = getQueryContextKey(
+    'durableShuffleStorage',
+    queryContext,
+    defaultQueryContext,
+  );
+
   const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec');
-  const useApproximateCountDistinct = getUseApproximateCountDistinct(queryContext);
-  const useApproximateTopN = getUseApproximateTopN(queryContext);
-  const useCache = getUseCache(queryContext);
-  const timezone = getTimezone(queryContext);
 
   const handleRun = useCallback(() => {
     if (!onRun) return;
@@ -210,7 +224,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
   const queryEngine = query.engine;
 
   function changeQueryContext(queryContext: QueryContext) {
-    onQueryChange(query.changeQueryContext(queryContext));
+    onQueryChange(query.changeQueryContext(removeUndefinedValues(queryContext)));
   }
 
   function offsetOptions(): JSX.Element[] {
@@ -221,10 +235,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
       items.push(
         <MenuItem
           key={offset}
-          icon={tickIcon(offset === timezone)}
+          icon={tickIcon(offset === sqlTimeZone)}
           text={offset}
           shouldDismissPopover={false}
-          onClick={() => changeQueryContext(changeTimezone(queryContext, offset))}
+          onClick={() => changeQueryContext({ ...queryContext, sqlTimeZone: offset })}
         />,
       );
     }
@@ -315,29 +329,32 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                   <MenuItem
                     icon={IconNames.GLOBE_NETWORK}
                     text="Timezone"
-                    label={timezone || 'default'}
+                    label={sqlTimeZone ?? defaultQueryContext.sqlTimeZone}
                   >
                     <MenuDivider title="Timezone type" />
                     <MenuItem
-                      icon={tickIcon(!timezone)}
+                      icon={tickIcon(!sqlTimeZone)}
                       text="Default"
+                      label={defaultQueryContext.sqlTimeZone}
                       shouldDismissPopover={false}
-                      onClick={() => changeQueryContext(changeTimezone(queryContext, undefined))}
+                      onClick={() =>
+                        changeQueryContext({ ...queryContext, sqlTimeZone: undefined })
+                      }
                     />
-                    <MenuItem icon={tickIcon(String(timezone).includes('/'))} text="Named">
+                    <MenuItem icon={tickIcon(String(sqlTimeZone).includes('/'))} text="Named">
                       {NAMED_TIMEZONES.map(namedTimezone => (
                         <MenuItem
                           key={namedTimezone}
-                          icon={tickIcon(namedTimezone === timezone)}
+                          icon={tickIcon(namedTimezone === sqlTimeZone)}
                           text={namedTimezone}
                           shouldDismissPopover={false}
                           onClick={() =>
-                            changeQueryContext(changeTimezone(queryContext, namedTimezone))
+                            changeQueryContext({ ...queryContext, sqlTimeZone: namedTimezone })
                           }
                         />
                       ))}
                     </MenuItem>
-                    <MenuItem icon={tickIcon(String(timezone).includes(':'))} text="Offset">
+                    <MenuItem icon={tickIcon(String(sqlTimeZone).includes(':'))} text="Offset">
                       {offsetOptions()}
                     </MenuItem>
                     <MenuItem
@@ -360,7 +377,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                           icon={tickIcon(v === maxParseExceptions)}
                           text={v === -1 ? '∞ (-1)' : String(v)}
                           onClick={() =>
-                            changeQueryContext(changeMaxParseExceptions(queryContext, v))
+                            changeQueryContext({ ...queryContext, maxParseExceptions: v })
                           }
                           shouldDismissPopover={false}
                         />
@@ -371,8 +388,8 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       text="Fail on empty insert"
                       value={failOnEmptyInsert}
                       undefinedEffectiveValue={false}
-                      onValueChange={v =>
-                        changeQueryContext(changeFailOnEmptyInsert(queryContext, v))
+                      onValueChange={failOnEmptyInsert =>
+                        changeQueryContext({ ...queryContext, failOnEmptyInsert })
                       }
                     />
                     <MenuTristate
@@ -380,8 +397,8 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       text="Finalize aggregations"
                       value={finalizeAggregations}
                       undefinedEffectiveValue={!ingestMode}
-                      onValueChange={v =>
-                        changeQueryContext(changeFinalizeAggregations(queryContext, v))
+                      onValueChange={finalizeAggregations =>
+                        changeQueryContext({ ...queryContext, finalizeAggregations })
                       }
                     />
                     <MenuTristate
@@ -389,8 +406,8 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       text="Wait until segments have loaded"
                       value={waitUntilSegmentsLoad}
                       undefinedEffectiveValue={ingestMode}
-                      onValueChange={v =>
-                        changeQueryContext(changeWaitUntilSegmentsLoad(queryContext, v))
+                      onValueChange={waitUntilSegmentsLoad =>
+                        changeQueryContext({ ...queryContext, waitUntilSegmentsLoad })
                       }
                     />
                     <MenuTristate
@@ -398,8 +415,8 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       text="Enable GroupBy multi-value unnesting"
                       value={groupByEnableMultiValueUnnesting}
                       undefinedEffectiveValue={!ingestMode}
-                      onValueChange={v =>
-                        changeQueryContext(changeGroupByEnableMultiValueUnnesting(queryContext, v))
+                      onValueChange={groupByEnableMultiValueUnnesting =>
+                        changeQueryContext({ ...queryContext, groupByEnableMultiValueUnnesting })
                       }
                     />
                     <MenuItem
@@ -407,14 +424,14 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       text="Join algorithm"
                       label={sqlJoinAlgorithm}
                     >
-                      {['broadcast', 'sortMerge'].map(o => (
+                      {(['broadcast', 'sortMerge'] as SqlJoinAlgorithm[]).map(o => (
                         <MenuItem
                           key={o}
                           icon={tickIcon(sqlJoinAlgorithm === o)}
                           text={o}
                           shouldDismissPopover={false}
                           onClick={() =>
-                            changeQueryContext(deepSet(queryContext, 'sqlJoinAlgorithm', o))
+                            changeQueryContext({ ...queryContext, sqlJoinAlgorithm: o })
                           }
                         />
                       ))}
@@ -425,14 +442,14 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       label={selectDestination}
                       intent={intent}
                     >
-                      {['taskReport', 'durableStorage'].map(o => (
+                      {(['taskReport', 'durableStorage'] as SelectDestination[]).map(o => (
                         <MenuItem
                           key={o}
                           icon={tickIcon(selectDestination === o)}
                           text={o}
                           shouldDismissPopover={false}
                           onClick={() =>
-                            changeQueryContext(deepSet(queryContext, 'selectDestination', o))
+                            changeQueryContext({ ...queryContext, selectDestination: o })
                           }
                         />
                       ))}
@@ -454,9 +471,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       checked={durableShuffleStorage}
                       text="Durable shuffle storage"
                       onChange={() =>
-                        changeQueryContext(
-                          changeDurableShuffleStorage(queryContext, !durableShuffleStorage),
-                        )
+                        changeQueryContext({
+                          ...queryContext,
+                          durableShuffleStorage: !durableShuffleStorage,
+                        })
                       }
                     />
                     <MenuItem
@@ -474,15 +492,22 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                     <MenuCheckbox
                       checked={useCache}
                       text="Use cache"
-                      onChange={() => changeQueryContext(changeUseCache(queryContext, !useCache))}
+                      onChange={() =>
+                        changeQueryContext({
+                          ...queryContext,
+                          useCache: !useCache,
+                          populateCache: !useCache,
+                        })
+                      }
                     />
                     <MenuCheckbox
                       checked={useApproximateTopN}
                       text="Use approximate TopN"
                       onChange={() =>
-                        changeQueryContext(
-                          changeUseApproximateTopN(queryContext, !useApproximateTopN),
-                        )
+                        changeQueryContext({
+                          ...queryContext,
+                          useApproximateTopN: !useApproximateTopN,
+                        })
                       }
                     />
                   </>
@@ -492,12 +517,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                     checked={useApproximateCountDistinct}
                     text="Use approximate COUNT(DISTINCT)"
                     onChange={() =>
-                      changeQueryContext(
-                        changeUseApproximateCountDistinct(
-                          queryContext,
-                          !useApproximateCountDistinct,
-                        ),
-                      )
+                      changeQueryContext({
+                        ...queryContext,
+                        useApproximateCountDistinct: !useApproximateCountDistinct,
+                      })
                     }
                   />
                 )}
@@ -519,8 +542,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
           >
             <Button
               text={`Engine: ${
-                (enginesLabelFn ? enginesLabelFn(queryEngine).text : queryEngine) ||
-                `auto (${enginesLabelFn ? enginesLabelFn(effectiveEngine) : effectiveEngine})`
+                queryEngine
+                  ? enginesLabelFn(queryEngine).text
+                  : `${autoEngineLabel.text} (${enginesLabelFn(effectiveEngine).text})`
               }`}
               rightIcon={IconNames.CARET_DOWN}
               intent={intent}
@@ -531,8 +555,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
               clusterCapacity={clusterCapacity}
               queryContext={queryContext}
               changeQueryContext={changeQueryContext}
-              menuHeader={maxTaskMenuHeader}
-              maxNumLabelFn={maxTaskLabelFn}
+              defaultQueryContext={defaultQueryContext}
+              menuHeader={maxTasksMenuHeader}
+              maxTasksLabelFn={maxTasksLabelFn}
+              fullClusterCapacityLabelFn={fullClusterCapacityLabelFn}
             />
           )}
           {ingestMode && (
@@ -544,8 +570,16 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                     <MenuItem
                       key={i}
                       icon={tickIcon(m === arrayIngestMode)}
-                      text={m ? ARRAY_INGEST_MODE_DESCRIPTION[m] : '(server default)'}
-                      onClick={() => changeQueryContext(changeArrayIngestMode(queryContext, m))}
+                      text={
+                        m
+                          ? ARRAY_INGEST_MODE_DESCRIPTION[m]
+                          : `(server default${
+                              defaultQueryContext.arrayIngestMode
+                                ? `: ${defaultQueryContext.arrayIngestMode}`
+                                : ''
+                            })`
+                      }
+                      onClick={() => changeQueryContext({ ...queryContext, arrayIngestMode: m })}
                     />
                   ))}
                   <MenuDivider />
@@ -594,7 +628,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
           title="Custom timezone"
           placeholder="Etc/UTC"
           maxLength={50}
-          onSubmit={tz => changeQueryContext(changeTimezone(queryContext, tz))}
+          onSubmit={sqlTimeZone => changeQueryContext({ ...queryContext, sqlTimeZone })}
           onClose={() => setCustomTimezoneDialogOpen(false)}
         />
       )}
diff --git a/web-console/src/views/workbench-view/workbench-view.tsx b/web-console/src/views/workbench-view/workbench-view.tsx
index aaac54e6d4a..e4f15a3aa18 100644
--- a/web-console/src/views/workbench-view/workbench-view.tsx
+++ b/web-console/src/views/workbench-view/workbench-view.tsx
@@ -30,7 +30,6 @@ import type { SqlQuery } from '@druid-toolkit/query';
 import { SqlExpression } from '@druid-toolkit/query';
 import classNames from 'classnames';
 import copy from 'copy-to-clipboard';
-import type { ComponentProps } from 'react';
 import React from 'react';
 
 import { SpecDialog, StringInputDialog } from '../../dialogs';
@@ -38,10 +37,15 @@ import type {
   CapacityInfo,
   DruidEngine,
   Execution,
+  QueryContext,
   QueryWithContext,
   TabEntry,
 } from '../../druid-models';
-import { guessDataSourceNameFromInputSource, WorkbenchQuery } from '../../druid-models';
+import {
+  DEFAULT_SERVER_QUERY_CONTEXT,
+  guessDataSourceNameFromInputSource,
+  WorkbenchQuery,
+} from '../../druid-models';
 import type { Capabilities } from '../../helpers';
 import { convertSpecToSql, getSpecDatasourceName, getTaskExecution } from '../../helpers';
 import { getLink } from '../../links';
@@ -71,6 +75,7 @@ import type { ExecutionDetailsTab } from './execution-details-pane/execution-det
 import { ExecutionSubmitDialog } from './execution-submit-dialog/execution-submit-dialog';
 import { ExplainDialog } from './explain-dialog/explain-dialog';
 import { MetadataChangeDetector } from './metadata-change-detector';
+import type { QueryTabProps } from './query-tab/query-tab';
 import { QueryTab } from './query-tab/query-tab';
 import { RecentQueryTaskPanel } from './recent-query-task-panel/recent-query-task-panel';
 import { TabRenameDialog } from './tab-rename-dialog/tab-rename-dialog';
@@ -91,20 +96,23 @@ function externalDataTabId(tabId: string | undefined): boolean {
   return String(tabId).startsWith('connect-external-data');
 }
 
-export interface WorkbenchViewProps {
+export interface WorkbenchViewProps
+  extends Pick<
+    QueryTabProps,
+    'maxTasksMenuHeader' | 'enginesLabelFn' | 'maxTasksLabelFn' | 'fullClusterCapacityLabelFn'
+  > {
   capabilities: Capabilities;
   tabId: string | undefined;
   onTabChange(newTabId: string): void;
   initQueryWithContext: QueryWithContext | undefined;
-  defaultQueryContext?: Record<string, any>;
-  mandatoryQueryContext?: Record<string, any>;
+  baseQueryContext?: QueryContext;
+  defaultQueryContext?: QueryContext;
+  mandatoryQueryContext?: QueryContext;
+  serverQueryContext?: QueryContext;
   queryEngines: DruidEngine[];
   allowExplain: boolean;
   goToTask(taskId: string): void;
   getClusterCapacity: (() => Promise<CapacityInfo | undefined>) | undefined;
-  maxTaskMenuHeader?: JSX.Element;
-  enginesLabelFn?: ComponentProps<typeof QueryTab>['enginesLabelFn'];
-  maxTaskLabelFn?: ComponentProps<typeof QueryTab>['maxTaskLabelFn'];
   hideToolbar?: boolean;
 }
 
@@ -249,11 +257,15 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
     });
   };
 
+  private getInitWorkbenchQuery(): WorkbenchQuery {
+    return WorkbenchQuery.blank().changeQueryContext(this.props.defaultQueryContext || {});
+  }
+
   private getInitTab(): TabEntry {
     return {
       id: generate8HexId(),
       tabName: 'Tab 1',
-      query: WorkbenchQuery.blank().changeQueryContext(this.props.defaultQueryContext || {}),
+      query: this.getInitWorkbenchQuery(),
     };
   }
 
@@ -607,7 +619,7 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
           icon={IconNames.PLUS}
           minimal
           onClick={() => {
-            this.handleNewTab(WorkbenchQuery.blank());
+            this.handleNewTab(this.getInitWorkbenchQuery());
           }}
         />
       </div>
@@ -651,13 +663,16 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
     const {
       capabilities,
       mandatoryQueryContext,
+      baseQueryContext,
+      serverQueryContext = DEFAULT_SERVER_QUERY_CONTEXT,
       queryEngines,
       allowExplain,
       goToTask,
       getClusterCapacity,
-      maxTaskMenuHeader,
+      maxTasksMenuHeader,
       enginesLabelFn,
-      maxTaskLabelFn,
+      maxTasksLabelFn,
+      fullClusterCapacityLabelFn,
     } = this.props;
     const { columnMetadataState } = this.state;
     const currentTabEntry = this.getCurrentTabEntry();
@@ -674,6 +689,8 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
           query={currentTabEntry.query}
           id={currentTabEntry.id}
           mandatoryQueryContext={mandatoryQueryContext}
+          baseQueryContext={baseQueryContext}
+          serverQueryContext={serverQueryContext}
           columnMetadata={columnMetadataState.getSomeData()}
           onQueryChange={this.handleQueryChange}
           onQueryTab={this.handleNewTab}
@@ -682,9 +699,10 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
           clusterCapacity={capabilities.getMaxTaskSlots()}
           goToTask={goToTask}
           getClusterCapacity={getClusterCapacity}
-          maxTaskMenuHeader={maxTaskMenuHeader}
+          maxTasksMenuHeader={maxTasksMenuHeader}
           enginesLabelFn={enginesLabelFn}
-          maxTaskLabelFn={maxTaskLabelFn}
+          maxTasksLabelFn={maxTasksLabelFn}
+          fullClusterCapacityLabelFn={fullClusterCapacityLabelFn}
           runMoreMenu={
             <Menu>
               {allowExplain &&

From 4ef4e75c5dfa4a49579f31e6d5af91437696077a Mon Sep 17 00:00:00 2001
From: Rushikesh Bankar <67940386+rbankar7@users.noreply.github.com>
Date: Sat, 10 Aug 2024 14:53:16 +0530
Subject: [PATCH 05/99] Fix the issue of missing implementation of
 IndexerTaskCountStatsProvider for peons (#16875)

Bug description:
Peons to fail to start up when `WorkerTaskCountStatsMonitor` is used on MiddleManagers.
This is because MiddleManagers pass on their properties to peons and peons are unable to
find `IndexerTaskCountStatsProvider` as that is bound only for indexer nodes.

Fix:
Check if node is an indexer before trying to get instance of `IndexerTaskCountStatsProvider`.
---
 .../server/metrics/WorkerTaskCountStatsMonitor.java   | 11 +++++++++--
 .../metrics/WorkerTaskCountStatsMonitorTest.java      | 11 +++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitor.java
index bc09e95b5ce..a568ad85d42 100644
--- a/server/src/main/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitor.java
+++ b/server/src/main/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitor.java
@@ -38,6 +38,7 @@ public class WorkerTaskCountStatsMonitor extends AbstractMonitor
   private final String workerCategory;
   private final String workerVersion;
   private final boolean isMiddleManager;
+  private final boolean isIndexer;
 
   @Inject
   public WorkerTaskCountStatsMonitor(
@@ -46,16 +47,22 @@ public class WorkerTaskCountStatsMonitor extends AbstractMonitor
   )
   {
     this.isMiddleManager = nodeRoles.contains(NodeRole.MIDDLE_MANAGER);
+    this.isIndexer = nodeRoles.contains(NodeRole.INDEXER);
     if (isMiddleManager) {
       this.statsProvider = injector.getInstance(WorkerTaskCountStatsProvider.class);
       this.indexerStatsProvider = null;
       this.workerCategory = statsProvider.getWorkerCategory();
       this.workerVersion = statsProvider.getWorkerVersion();
-    } else {
+    } else if (isIndexer) {
       this.indexerStatsProvider = injector.getInstance(IndexerTaskCountStatsProvider.class);
       this.statsProvider = null;
       this.workerCategory = null;
       this.workerVersion = null;
+    } else {
+      this.indexerStatsProvider = null;
+      this.statsProvider = null;
+      this.workerCategory = null;
+      this.workerVersion = null;
     }
   }
 
@@ -68,7 +75,7 @@ public class WorkerTaskCountStatsMonitor extends AbstractMonitor
       emit(emitter, "worker/taskSlot/idle/count", statsProvider.getWorkerIdleTaskSlotCount());
       emit(emitter, "worker/taskSlot/total/count", statsProvider.getWorkerTotalTaskSlotCount());
       emit(emitter, "worker/taskSlot/used/count", statsProvider.getWorkerUsedTaskSlotCount());
-    } else {
+    } else if (isIndexer) {
       emit(emitter, "worker/task/running/count", indexerStatsProvider.getWorkerRunningTasks());
       emit(emitter, "worker/task/assigned/count", indexerStatsProvider.getWorkerAssignedTasks());
       emit(emitter, "worker/task/completed/count", indexerStatsProvider.getWorkerCompletedTasks());
diff --git a/server/src/test/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitorTest.java b/server/src/test/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitorTest.java
index ad00e5e6dbd..a1930bef637 100644
--- a/server/src/test/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitorTest.java
+++ b/server/src/test/java/org/apache/druid/server/metrics/WorkerTaskCountStatsMonitorTest.java
@@ -309,6 +309,7 @@ public class WorkerTaskCountStatsMonitorTest
             89L
     );
   }
+
   @Test
   public void testMonitorWithNulls()
   {
@@ -318,4 +319,14 @@ public class WorkerTaskCountStatsMonitorTest
     monitor.doMonitor(emitter);
     Assert.assertEquals(0, emitter.getEvents().size());
   }
+
+  @Test
+  public void testMonitorWithPeon()
+  {
+    final WorkerTaskCountStatsMonitor monitor =
+            new WorkerTaskCountStatsMonitor(injectorForPeon, ImmutableSet.of(NodeRole.PEON));
+    final StubServiceEmitter emitter = new StubServiceEmitter("service", "host");
+    monitor.doMonitor(emitter);
+    Assert.assertEquals(0, emitter.getEvents().size());
+  }
 }

From efe0044f9e38e494d75ddd1fe432d601b24c4b75 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Sun, 11 Aug 2024 19:00:33 -0700
Subject: [PATCH 06/99] Use fuzzy matchers for compaction bytes asserts.
 (#16870)

* Use fuzzy matchers for compaction bytes asserts.

This still enables us to test that the bytes are zero and nonzero
when they're supposed to be, without having to ge them exactly
right. The need to get bytes exactly right makes it difficult to
ensure ITs pass when making changes to default segment metadata.

* Additional fuzziness.
---
 integration-tests/pom.xml                     | 10 +++++
 .../duty/ITAutoCompactionTest.java            | 44 +++++++++++--------
 2 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
index a6600df02b7..e93fb7464e3 100644
--- a/integration-tests/pom.xml
+++ b/integration-tests/pom.xml
@@ -430,6 +430,16 @@
             <artifactId>avatica-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-core</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
index 8f070f33405..24ede82d0de 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
@@ -70,6 +70,9 @@ import org.apache.druid.tests.TestNGGroup;
 import org.apache.druid.tests.indexer.AbstractITBatchIndexTest;
 import org.apache.druid.tests.indexer.AbstractIndexerTest;
 import org.apache.druid.timeline.DataSegment;
+import org.hamcrest.Matcher;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Interval;
 import org.joda.time.Period;
@@ -532,9 +535,9 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
       getAndAssertCompactionStatus(
           fullDatasourceName,
           AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING,
-          0,
-          14166,
-          14165,
+          Matchers.equalTo(0L),
+          Matchers.greaterThan(0L),
+          Matchers.greaterThan(0L),
           0,
           2,
           2,
@@ -550,9 +553,9 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
       getAndAssertCompactionStatus(
           fullDatasourceName,
           AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING,
-          0,
-          22262,
-          0,
+          Matchers.equalTo(0L),
+          Matchers.greaterThan(0L),
+          Matchers.equalTo(0L),
           0,
           3,
           0,
@@ -670,16 +673,19 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
       getAndAssertCompactionStatus(
           fullDatasourceName,
           AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING,
-          14166,
-          14165,
-          0,
+          Matchers.greaterThan(0L),
+          Matchers.greaterThan(0L),
+          Matchers.equalTo(0L),
           2,
           2,
           0,
           1,
           1,
           0);
-      Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "14166");
+      MatcherAssert.assertThat(
+          Long.parseLong(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize")),
+          Matchers.greaterThan(0L)
+      );
       // Run compaction again to compact the remaining day
       // Remaining day compacted (1 new segment). Now both days compacted (2 total)
       forceTriggerAutoCompaction(2);
@@ -689,9 +695,9 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
       getAndAssertCompactionStatus(
           fullDatasourceName,
           AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING,
-          0,
-          22262,
-          0,
+          Matchers.equalTo(0L),
+          Matchers.greaterThan(0L),
+          Matchers.equalTo(0L),
           0,
           3,
           0,
@@ -1952,9 +1958,9 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
   private void getAndAssertCompactionStatus(
       String fullDatasourceName,
       AutoCompactionSnapshot.AutoCompactionScheduleStatus scheduleStatus,
-      long bytesAwaitingCompaction,
-      long bytesCompacted,
-      long bytesSkipped,
+      Matcher<Long> bytesAwaitingCompactionMatcher,
+      Matcher<Long> bytesCompactedMatcher,
+      Matcher<Long> bytesSkippedMatcher,
       long segmentCountAwaitingCompaction,
       long segmentCountCompacted,
       long segmentCountSkipped,
@@ -1966,9 +1972,9 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
     Map<String, String> actualStatus = compactionResource.getCompactionStatus(fullDatasourceName);
     Assert.assertNotNull(actualStatus);
     Assert.assertEquals(actualStatus.get("scheduleStatus"), scheduleStatus.toString());
-    Assert.assertEquals(Long.parseLong(actualStatus.get("bytesAwaitingCompaction")), bytesAwaitingCompaction);
-    Assert.assertEquals(Long.parseLong(actualStatus.get("bytesCompacted")), bytesCompacted);
-    Assert.assertEquals(Long.parseLong(actualStatus.get("bytesSkipped")), bytesSkipped);
+    MatcherAssert.assertThat(Long.parseLong(actualStatus.get("bytesAwaitingCompaction")), bytesAwaitingCompactionMatcher);
+    MatcherAssert.assertThat(Long.parseLong(actualStatus.get("bytesCompacted")), bytesCompactedMatcher);
+    MatcherAssert.assertThat(Long.parseLong(actualStatus.get("bytesSkipped")), bytesSkippedMatcher);
     Assert.assertEquals(Long.parseLong(actualStatus.get("segmentCountAwaitingCompaction")), segmentCountAwaitingCompaction);
     Assert.assertEquals(Long.parseLong(actualStatus.get("segmentCountCompacted")), segmentCountCompacted);
     Assert.assertEquals(Long.parseLong(actualStatus.get("segmentCountSkipped")), segmentCountSkipped);

From d7dfbebf974f3a5e9a4d68a3a78c6b22eea9cd33 Mon Sep 17 00:00:00 2001
From: Abhishek Radhakrishnan <abhishek.rb19@gmail.com>
Date: Mon, 12 Aug 2024 13:55:33 -0700
Subject: [PATCH 07/99] [Docs]: Fix typo and update broadcast rules section
 (#16882)

* Fix typo in waitUntilSegmentsLoad.

* Add a note on configuring druid.segmentCache.locations for broadcast rules.

* Update docs/operations/rule-configuration.md

Co-authored-by: Victoria Lim <vtlim@users.noreply.github.com>

---------

Co-authored-by: Victoria Lim <vtlim@users.noreply.github.com>
---
 docs/multi-stage-query/reference.md   | 2 +-
 docs/operations/rule-configuration.md | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/multi-stage-query/reference.md b/docs/multi-stage-query/reference.md
index cf06156c658..0f8e710a59f 100644
--- a/docs/multi-stage-query/reference.md
+++ b/docs/multi-stage-query/reference.md
@@ -404,7 +404,7 @@ The following table lists the context parameters for the MSQ task engine:
 | `durableShuffleStorage` | SELECT, INSERT, REPLACE <br /><br />Whether to use durable storage for shuffle mesh. To use this feature, configure the durable storage at the server level using `druid.msq.intermediate.storage.enable=true`). If these properties are not configured, any query with the context variable `durableShuffleStorage=true` fails with a configuration error. <br /><br /> | `false` |
 | `faultTolerance` | SELECT, INSERT, REPLACE<br /><br /> Whether to turn on fault tolerance mode or not. Failed workers are retried based on [Limits](#limits). Cannot be used when `durableShuffleStorage` is explicitly set to false. | `false` |
 | `selectDestination` | SELECT<br /><br /> Controls where the final result of the select query is written. <br />Use `taskReport`(the default) to write select results to the task report. <b> This is not scalable since task reports size explodes for large results </b> <br/>Use `durableStorage` to write results to durable storage location. <b>For large results sets, its recommended to use `durableStorage` </b>. To configure durable storage see [`this`](#durable-storage) section. | `taskReport` |
-| `waitUntilSegmentsLoad` | INSERT, REPLACE<br /><br /> If set, the ingest query waits for the generated segment to be loaded before exiting, else the ingest query exits without waiting. The task and live reports contain the information about the status of loading segments if this flag is set. This will ensure that any future queries made after the ingestion exits will include results from the ingestion. The drawback is that the controller task will stall till the segments are loaded. | `false` |
+| `waitUntilSegmentsLoad` | INSERT, REPLACE<br /><br /> If set, the ingest query waits for the generated segments to be loaded before exiting, else the ingest query exits without waiting. The task and live reports contain the information about the status of loading segments if this flag is set. This will ensure that any future queries made after the ingestion exits will include results from the ingestion. The drawback is that the controller task will stall till the segments are loaded. | `false` |
 | `includeSegmentSource` | SELECT, INSERT, REPLACE<br /><br /> Controls the sources, which will be queried for results in addition to the segments present on deep storage. Can be `NONE` or `REALTIME`. If this value is `NONE`, only non-realtime (published and used) segments will be downloaded from deep storage. If this value is `REALTIME`, results will also be included from realtime tasks. `REALTIME` cannot be used while writing data into the same datasource it is read from.| `NONE` |
 | `rowsPerPage` | SELECT<br /><br />The number of rows per page to target. The actual number of rows per page may be somewhat higher or lower than this number. In most cases, use the default.<br /> This property comes into effect only when `selectDestination` is set to `durableStorage` | 100000 |
 | `skipTypeVerification` | INSERT or REPLACE<br /><br />During query validation, Druid validates that [string arrays](../querying/arrays.md) and [multi-value dimensions](../querying/multi-value-dimensions.md) are not mixed in the same column. If you are intentionally migrating from one to the other, use this context parameter to disable type validation.<br /><br />Provide the column list as comma-separated values or as a JSON array in string form.| empty list |
diff --git a/docs/operations/rule-configuration.md b/docs/operations/rule-configuration.md
index 610c2fa6dbf..9117973f0b8 100644
--- a/docs/operations/rule-configuration.md
+++ b/docs/operations/rule-configuration.md
@@ -277,7 +277,9 @@ Set the following property:
 
 ## Broadcast rules
 
-Druid extensions use broadcast rules to load segment data onto all brokers in the cluster. Apply broadcast rules in a test environment, not in production.
+Druid extensions use broadcast rules to load segment data onto all Brokers in the cluster. Apply broadcast rules in a test environment, not in production.
+To use broadcast rules, ensure that `druid.segmentCache.locations` is configured on both Brokers and Historicals.
+This ensures that Druid can load the segments onto those servers. For more information, see [Segment cache size](../operations/basic-cluster-tuning.md#segment-cache-size).
 
 ### Forever broadcast rule
 

From f67ff92d07d4838e6acd5fb4761baf3c9de6c3e2 Mon Sep 17 00:00:00 2001
From: Rishabh Singh <6513075+findingrish@users.noreply.github.com>
Date: Tue, 13 Aug 2024 11:44:01 +0530
Subject: [PATCH 08/99] [bugfix] Run cold schema refresh thread periodically 
 (#16873)

* Fix build

* Run coldSchemaExec thread periodically

* Bugfix: Run cold schema refresh periodically

* Rename metrics for deep storage only segment schema process
---
 docs/operations/metrics.md                    |  3 +
 .../CoordinatorSegmentMetadataCache.java      | 82 +++++++++++++------
 .../CoordinatorSegmentMetadataCacheTest.java  | 64 +++++++++++++--
 3 files changed, 121 insertions(+), 28 deletions(-)

diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md
index ec97f44fe39..83cc9550690 100644
--- a/docs/operations/metrics.md
+++ b/docs/operations/metrics.md
@@ -382,6 +382,9 @@ These metrics are emitted by the Druid Coordinator in every run of the correspon
 |`metadatacache/finalizedSchemaPayload/count`|Number of finalized segment schema cached.||Depends on the number of distinct schema in the cluster.|
 |`metadatacache/temporaryMetadataQueryResults/count`|Number of segments for which schema was fetched by executing segment metadata query.||Eventually it should be 0.|
 |`metadatacache/temporaryPublishedMetadataQueryResults/count`|Number of segments for which schema is cached after back filling in the database.||This value gets reset after each database poll. Eventually it should be 0.|
+|`metadatacache/deepStorageOnly/segment/count`|Number of available segments present only in deep storage.|`dataSource`||
+|`metadatacache/deepStorageOnly/refresh/count`|Number of deep storage only segments with cached schema.|`dataSource`||
+|`metadatacache/deepStorageOnly/process/time`|Time taken in milliseconds to process deep storage only segment schema.||Under a minute|
 
 ## General Health
 
diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
index 3a4f548b8ba..0c03f7af73c 100644
--- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
@@ -24,7 +24,6 @@ import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.inject.Inject;
 import org.apache.druid.client.CoordinatorServerView;
 import org.apache.druid.client.ImmutableDruidDataSource;
@@ -35,12 +34,15 @@ import org.apache.druid.guice.ManageLifecycle;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.Stopwatch;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.lifecycle.LifecycleStart;
 import org.apache.druid.java.util.common.lifecycle.LifecycleStop;
 import org.apache.druid.java.util.emitter.EmittingLogger;
 import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
 import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
 import org.apache.druid.metadata.SqlSegmentsMetadataManager;
+import org.apache.druid.query.DruidMetrics;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.metadata.metadata.SegmentAnalysis;
 import org.apache.druid.segment.SchemaPayloadPlus;
@@ -69,7 +71,6 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -100,12 +101,15 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
   private static final EmittingLogger log = new EmittingLogger(CoordinatorSegmentMetadataCache.class);
   private static final Long COLD_SCHEMA_PERIOD_MULTIPLIER = 3L;
   private static final Long COLD_SCHEMA_SLOWNESS_THRESHOLD_MILLIS = TimeUnit.SECONDS.toMillis(50);
+  private static final String DEEP_STORAGE_ONLY_METRIC_PREFIX = "metadatacache/deepStorageOnly/";
 
   private final SegmentMetadataCacheConfig config;
   private final ColumnTypeMergePolicy columnTypeMergePolicy;
   private final SegmentSchemaCache segmentSchemaCache;
   private final SegmentSchemaBackFillQueue segmentSchemaBackfillQueue;
   private final SqlSegmentsMetadataManager sqlSegmentsMetadataManager;
+  private final Supplier<SegmentsMetadataManagerConfig> segmentsMetadataManagerConfigSupplier;
+  private final ServiceEmitter emitter;
   private volatile SegmentReplicationStatus segmentReplicationStatus = null;
 
   // Datasource schema built from only cold segments.
@@ -114,7 +118,6 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
   // Period for cold schema processing thread. This is a multiple of segment polling period.
   // Cold schema processing runs slower than the segment poll to save processing cost of all segments.
   // The downside is a delay in columns from cold segment reflecting in the datasource schema.
-  private final long coldSchemaExecPeriodMillis;
   private final ScheduledExecutorService coldSchemaExec;
   private @Nullable Future<?> cacheExecFuture = null;
   private @Nullable Future<?> coldSchemaExecFuture = null;
@@ -139,18 +142,19 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
     this.segmentSchemaCache = segmentSchemaCache;
     this.segmentSchemaBackfillQueue = segmentSchemaBackfillQueue;
     this.sqlSegmentsMetadataManager = sqlSegmentsMetadataManager;
-    this.coldSchemaExecPeriodMillis =
-        segmentsMetadataManagerConfigSupplier.get().getPollDuration().getMillis() * COLD_SCHEMA_PERIOD_MULTIPLIER;
-    coldSchemaExec = Executors.newSingleThreadScheduledExecutor(
-        new ThreadFactoryBuilder()
-            .setNameFormat("DruidColdSchema-ScheduledExecutor-%d")
-            .setDaemon(false)
-            .build()
-    );
+    this.segmentsMetadataManagerConfigSupplier = segmentsMetadataManagerConfigSupplier;
+    this.emitter = emitter;
+    this.coldSchemaExec = Execs.scheduledSingleThreaded("DruidColdSchema-ScheduledExecutor-%d");
 
     initServerViewTimelineCallback(serverView);
   }
 
+  long getColdSchemaExecPeriodMillis()
+  {
+    return (segmentsMetadataManagerConfigSupplier.get().getPollDuration().toStandardDuration().getMillis())
+           * COLD_SCHEMA_PERIOD_MULTIPLIER;
+  }
+
   private void initServerViewTimelineCallback(final CoordinatorServerView serverView)
   {
     serverView.registerTimelineCallback(
@@ -232,9 +236,10 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
     try {
       segmentSchemaBackfillQueue.onLeaderStart();
       cacheExecFuture = cacheExec.submit(this::cacheExecLoop);
-      coldSchemaExecFuture = coldSchemaExec.schedule(
+      coldSchemaExecFuture = coldSchemaExec.scheduleWithFixedDelay(
           this::coldDatasourceSchemaExec,
-          coldSchemaExecPeriodMillis,
+          getColdSchemaExecPeriodMillis(),
+          getColdSchemaExecPeriodMillis(),
           TimeUnit.MILLISECONDS
       );
 
@@ -558,9 +563,7 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
 
     Set<String> dataSourceWithColdSegmentSet = new HashSet<>();
 
-    int datasources = 0;
-    int segments = 0;
-    int dataSourceWithColdSegments = 0;
+    int datasources = 0, dataSourceWithColdSegments = 0, totalColdSegments = 0;
 
     Collection<ImmutableDruidDataSource> immutableDataSources =
         sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments();
@@ -571,6 +574,9 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
 
       final Map<String, ColumnType> columnTypes = new LinkedHashMap<>();
 
+      int coldSegments = 0;
+      int coldSegmentWithSchema = 0;
+
       for (DataSegment segment : dataSegments) {
         Integer replicationFactor = getReplicationFactor(segment.getId());
         if (replicationFactor != null && replicationFactor != 0) {
@@ -580,36 +586,66 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
         if (optionalSchema.isPresent()) {
           RowSignature rowSignature = optionalSchema.get().getSchemaPayload().getRowSignature();
           mergeRowSignature(columnTypes, rowSignature);
+          coldSegmentWithSchema++;
         }
-        segments++;
+        coldSegments++;
       }
 
-      if (columnTypes.isEmpty()) {
+      if (coldSegments == 0) {
         // this datasource doesn't have any cold segment
         continue;
       }
 
+      totalColdSegments += coldSegments;
+
+      String dataSourceName = dataSource.getName();
+
+      ServiceMetricEvent.Builder metricBuilder =
+          new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSourceName);
+
+      emitter.emit(metricBuilder.setMetric(DEEP_STORAGE_ONLY_METRIC_PREFIX + "segment/count", coldSegments));
+
+      if (columnTypes.isEmpty()) {
+        // this datasource doesn't have schema for cold segments
+        continue;
+      }
+
       final RowSignature.Builder builder = RowSignature.builder();
       columnTypes.forEach(builder::add);
 
       RowSignature coldSignature = builder.build();
 
-      String dataSourceName = dataSource.getName();
       dataSourceWithColdSegmentSet.add(dataSourceName);
       dataSourceWithColdSegments++;
 
-      log.debug("[%s] signature from cold segments is [%s]", dataSourceName, coldSignature);
+      DataSourceInformation druidTable = new DataSourceInformation(dataSourceName, coldSignature);
+      DataSourceInformation oldTable = coldSchemaTable.put(dataSourceName, druidTable);
 
-      coldSchemaTable.put(dataSourceName, new DataSourceInformation(dataSourceName, coldSignature));
+      if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) {
+        log.info("[%s] has new cold signature: %s.", dataSource, druidTable.getRowSignature());
+      } else {
+        log.debug("[%s] signature is unchanged.", dataSource);
+      }
+
+      emitter.emit(metricBuilder.setMetric(DEEP_STORAGE_ONLY_METRIC_PREFIX + "refresh/count", coldSegmentWithSchema));
+
+      log.debug("[%s] signature from cold segments is [%s]", dataSourceName, coldSignature);
     }
 
     // remove any stale datasource from the map
     coldSchemaTable.keySet().retainAll(dataSourceWithColdSegmentSet);
 
+    emitter.emit(
+        new ServiceMetricEvent.Builder().setMetric(
+            DEEP_STORAGE_ONLY_METRIC_PREFIX + "process/time",
+            stopwatch.millisElapsed()
+        )
+    );
+
     String executionStatsLog = StringUtils.format(
         "Cold schema processing took [%d] millis. "
-        + "Processed total [%d] datasources, [%d] segments. Found [%d] datasources with cold segments.",
-        stopwatch.millisElapsed(), datasources, segments, dataSourceWithColdSegments
+        + "Processed total [%d] datasources, [%d] segments. Found [%d] datasources with cold segment schema.",
+        stopwatch.millisElapsed(), datasources, totalColdSegments, dataSourceWithColdSegments
     );
     if (stopwatch.millisElapsed() > COLD_SCHEMA_SLOWNESS_THRESHOLD_MILLIS) {
       log.info(executionStatsLog);
diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
index ef1fb1e8edd..8fbc78a7412 100644
--- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
+++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
@@ -38,6 +38,7 @@ import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.concurrent.ScheduledExecutors;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.emitter.EmittingLogger;
+import org.apache.druid.java.util.emitter.service.ServiceEmitter;
 import org.apache.druid.java.util.metrics.StubServiceEmitter;
 import org.apache.druid.metadata.MetadataStorageTablesConfig;
 import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
@@ -1788,7 +1789,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
     Assert.assertEquals(existingMetadata.getNumReplicas(), currentMetadata.getNumReplicas());
   }
 
-  private CoordinatorSegmentMetadataCache setupForColdDatasourceSchemaTest()
+  private CoordinatorSegmentMetadataCache setupForColdDatasourceSchemaTest(ServiceEmitter emitter)
   {
     // foo has both hot and cold segments
     DataSegment coldSegment =
@@ -1862,7 +1863,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
         SEGMENT_CACHE_CONFIG_DEFAULT,
         new NoopEscalator(),
         new InternalQueryConfig(),
-        new NoopServiceEmitter(),
+        emitter,
         segmentSchemaCache,
         backFillQueue,
         sqlSegmentsMetadataManager,
@@ -1893,10 +1894,17 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
   @Test
   public void testColdDatasourceSchema_refreshAfterColdSchemaExec() throws IOException
   {
-    CoordinatorSegmentMetadataCache schema = setupForColdDatasourceSchemaTest();
+    StubServiceEmitter emitter = new StubServiceEmitter("coordinator", "host");
+    CoordinatorSegmentMetadataCache schema = setupForColdDatasourceSchemaTest(emitter);
 
     schema.coldDatasourceSchemaExec();
 
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/segment/count", ImmutableMap.of(DruidMetrics.DATASOURCE, "foo"), 1);
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, "foo"), 1);
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/segment/count", ImmutableMap.of(DruidMetrics.DATASOURCE, "cold"), 1);
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, "cold"), 1);
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/process/time", 1);
+
     Assert.assertEquals(new HashSet<>(Arrays.asList("foo", "cold")), schema.getDataSourceInformationMap().keySet());
 
     // verify that cold schema for both foo and cold is present
@@ -1955,7 +1963,8 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
   @Test
   public void testColdDatasourceSchema_coldSchemaExecAfterRefresh() throws IOException
   {
-    CoordinatorSegmentMetadataCache schema = setupForColdDatasourceSchemaTest();
+    StubServiceEmitter emitter = new StubServiceEmitter("coordinator", "host");
+    CoordinatorSegmentMetadataCache schema = setupForColdDatasourceSchemaTest(emitter);
 
     Set<SegmentId> segmentIds = new HashSet<>();
     segmentIds.add(segment1.getId());
@@ -1971,7 +1980,13 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
 
     schema.coldDatasourceSchemaExec();
 
-    // could datasource should be present now
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/segment/count", ImmutableMap.of(DruidMetrics.DATASOURCE, "foo"), 1);
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, "foo"), 1);
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/segment/count", ImmutableMap.of(DruidMetrics.DATASOURCE, "cold"), 1);
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, "cold"), 1);
+    emitter.verifyEmitted("metadatacache/deepStorageOnly/process/time", 1);
+
+    // cold datasource should be present now
     Assert.assertEquals(new HashSet<>(Arrays.asList("foo", "cold")), schema.getDataSourceInformationMap().keySet());
 
     RowSignature coldSignature = schema.getDatasource("cold").getRowSignature();
@@ -2160,6 +2175,45 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
     Assert.assertEquals(Collections.singleton("beta"), schema.getDataSourceInformationMap().keySet());
   }
 
+  @Test
+  public void testColdDatasourceSchemaExecRunsPeriodically() throws InterruptedException
+  {
+    // Make sure the thread runs more than once
+    CountDownLatch latch = new CountDownLatch(2);
+
+    CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache(
+        getQueryLifecycleFactory(walker),
+        serverView,
+        SEGMENT_CACHE_CONFIG_DEFAULT,
+        new NoopEscalator(),
+        new InternalQueryConfig(),
+        new NoopServiceEmitter(),
+        segmentSchemaCache,
+        backFillQueue,
+        sqlSegmentsMetadataManager,
+        segmentsMetadataManagerConfigSupplier
+    ) {
+      @Override
+      long getColdSchemaExecPeriodMillis()
+      {
+        return 10;
+      }
+
+      @Override
+      protected void coldDatasourceSchemaExec()
+      {
+        latch.countDown();
+        super.coldDatasourceSchemaExec();
+      }
+    };
+
+    schema.onLeaderStart();
+    schema.awaitInitialization();
+
+    latch.await(1, TimeUnit.SECONDS);
+    Assert.assertEquals(0, latch.getCount());
+  }
+
   private void verifyFooDSSchema(CoordinatorSegmentMetadataCache schema, int columns)
   {
     final DataSourceInformation fooDs = schema.getDatasource("foo");

From c6da2f30e8f33c570173b7ac649a829173187481 Mon Sep 17 00:00:00 2001
From: Adarsh Sanjeev <adarshsanjeev@gmail.com>
Date: Tue, 13 Aug 2024 14:04:41 +0530
Subject: [PATCH 09/99] Add fieldReader for row based frames (#16707)

Add a new fieldReaders#makeRAC for RowBasedFrameRowsAndColumns.
---
 .../druid/frame/field/ComplexFieldReader.java |  97 ++++-
 .../frame/field/DoubleArrayFieldReader.java   |  10 +
 .../druid/frame/field/DoubleFieldReader.java  | 140 +++++++
 .../frame/field/FieldPositionHelper.java      |  65 +++
 .../apache/druid/frame/field/FieldReader.java |  12 +-
 .../frame/field/FloatArrayFieldReader.java    |  10 +
 .../druid/frame/field/FloatFieldReader.java   | 140 +++++++
 .../frame/field/LongArrayFieldReader.java     |  10 +
 .../druid/frame/field/LongFieldReader.java    | 141 ++++++-
 .../frame/field/NumericArrayFieldReader.java  |   2 +-
 .../druid/frame/field/NumericFieldReader.java |   7 +-
 .../druid/frame/field/StringFieldReader.java  | 387 ++++++++++++++----
 .../concrete/RowBasedFrameRowsAndColumns.java |  17 +-
 .../frame/field/DoubleFieldReaderTest.java    |  30 ++
 .../druid/frame/field/FieldReaderRACTest.java |  80 ++++
 .../frame/field/FloatFieldReaderTest.java     |  30 ++
 .../frame/field/LongFieldReaderTest.java      |  31 ++
 .../rowsandcols/RowsAndColumnsTestBase.java   |   5 +-
 .../ColumnBasedFrameRowsAndColumnsTest.java   |   6 +-
 .../RowBasedFrameRowsAndColumnsTest.java      |  80 ++++
 20 files changed, 1210 insertions(+), 90 deletions(-)
 create mode 100644 processing/src/main/java/org/apache/druid/frame/field/FieldPositionHelper.java
 create mode 100644 processing/src/test/java/org/apache/druid/frame/field/FieldReaderRACTest.java
 create mode 100644 processing/src/test/java/org/apache/druid/query/rowsandcols/concrete/RowBasedFrameRowsAndColumnsTest.java

diff --git a/processing/src/main/java/org/apache/druid/frame/field/ComplexFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/ComplexFieldReader.java
index 29bf0945adb..75ad70d5cb4 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/ComplexFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/ComplexFieldReader.java
@@ -21,24 +21,32 @@ package org.apache.druid.frame.field;
 
 import com.google.common.base.Preconditions;
 import org.apache.datasketches.memory.Memory;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.write.RowBasedFrameWriter;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.extraction.ExtractionFn;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.query.rowsandcols.column.Column;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
+import org.apache.druid.query.rowsandcols.column.accessor.ObjectColumnAccessorBase;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.ObjectColumnSelector;
 import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
 import org.apache.druid.segment.serde.ComplexMetrics;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.Comparator;
 
 /**
  * Reads values written by {@link ComplexFieldWriter}.
- *
+ * <p>
  * Format:
- *
+ * <p>
  * - 1 byte: {@link ComplexFieldWriter#NULL_BYTE} or {@link ComplexFieldWriter#NOT_NULL_BYTE}
  * - 4 bytes: length of serialized complex value, little-endian int
  * - N bytes: serialized complex value
@@ -121,7 +129,7 @@ public class ComplexFieldReader implements FieldReader
    * Alternative interface to read the field from the memory without creating a selector and field pointer
    */
   @Nullable
-  public static Object readFieldFromMemory(
+  public static <T> T readFieldFromMemory(
       final ComplexMetricSerde serde,
       final Memory memory,
       final long position
@@ -136,7 +144,8 @@ public class ComplexFieldReader implements FieldReader
       final byte[] bytes = new byte[length];
       memory.getByteArray(position + ComplexFieldWriter.HEADER_SIZE, bytes, 0, length);
 
-      return serde.fromBytes(bytes, 0, length);
+      //noinspection unchecked
+      return (T) serde.fromBytes(bytes, 0, length);
     } else {
       throw new ISE("Unexpected null byte [%s]", nullByte);
     }
@@ -166,8 +175,8 @@ public class ComplexFieldReader implements FieldReader
     @Override
     public T getObject()
     {
-      //noinspection unchecked
-      return (T) readFieldFromMemory(serde, memory, fieldPointer.position());
+      final long fieldPosition = fieldPointer.position();
+      return readFieldFromMemory(serde, memory, fieldPosition);
     }
 
     @Override
@@ -183,4 +192,80 @@ public class ComplexFieldReader implements FieldReader
       // Do nothing.
     }
   }
+
+  @Override
+  public Column makeRACColumn(Frame frame, RowSignature signature, String columnName)
+  {
+    return new ComplexFieldReaderColumn(frame, signature.indexOf(columnName), signature.size());
+  }
+
+  private class ComplexFieldReaderColumn implements Column
+  {
+    private final Frame frame;
+    private final Memory dataRegion;
+    private final ColumnType type;
+    private final FieldPositionHelper coach;
+
+    public ComplexFieldReaderColumn(Frame frame, int columnIndex, int numFields)
+    {
+      this.frame = frame;
+      dataRegion = frame.region(RowBasedFrameWriter.ROW_DATA_REGION);
+
+      this.type = ColumnType.ofComplex(serde.getTypeName());
+      this.coach = new FieldPositionHelper(
+          frame,
+          frame.region(RowBasedFrameWriter.ROW_OFFSET_REGION),
+          dataRegion,
+          columnIndex,
+          numFields
+      );
+    }
+
+    @Nonnull
+    @Override
+    public ColumnAccessor toAccessor()
+    {
+      return new ObjectColumnAccessorBase()
+      {
+        @Override
+        public ColumnType getType()
+        {
+          return type;
+        }
+
+        @Override
+        public int numRows()
+        {
+          return frame.numRows();
+        }
+
+        @Override
+        public boolean isNull(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+          return dataRegion.getByte(fieldPosition) == ComplexFieldWriter.NULL_BYTE;
+        }
+
+        @Override
+        protected Object getVal(int rowNum)
+        {
+          return readFieldFromMemory(serde, dataRegion, coach.computeFieldPosition(rowNum));
+        }
+
+        @Override
+        protected Comparator<Object> getComparator()
+        {
+          return serde.getTypeStrategy();
+        }
+
+      };
+    }
+
+    @Nullable
+    @Override
+    public <T> T as(Class<? extends T> clazz)
+    {
+      return null;
+    }
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java
index ec7de095e12..2e57a3e9a5c 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java
@@ -20,7 +20,11 @@
 package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.Memory;
+import org.apache.druid.error.NotYetImplemented;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.query.rowsandcols.column.Column;
 import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.column.RowSignature;
 
 import javax.annotation.Nullable;
 
@@ -60,4 +64,10 @@ public class DoubleArrayFieldReader extends NumericArrayFieldReader
       }
     };
   }
+
+  @Override
+  public Column makeRACColumn(Frame frame, RowSignature signature, String columnName)
+  {
+    throw NotYetImplemented.ex(null, "Class cannot create an RAC column.");
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/DoubleFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/DoubleFieldReader.java
index 7f7a3f8639e..3afadebe711 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/DoubleFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/DoubleFieldReader.java
@@ -21,11 +21,20 @@ package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.Memory;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.write.RowBasedFrameWriter;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.query.rowsandcols.column.Column;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.DoubleColumnSelector;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 /**
  * Reads the values produced by {@link DoubleFieldWriter}
  *
@@ -99,4 +108,135 @@ public class DoubleFieldReader extends NumericFieldReader
       return super._isNull();
     }
   }
+
+  @Override
+  public Column makeRACColumn(Frame frame, RowSignature signature, String columnName)
+  {
+    return new DoubleFieldReaderColumn(frame, signature.indexOf(columnName), signature.size());
+  }
+
+  private class DoubleFieldReaderColumn implements Column
+  {
+    private final Frame frame;
+    private final Memory dataRegion;
+    private final FieldPositionHelper coach;
+
+    public DoubleFieldReaderColumn(Frame frame, int columnIndex, int numFields)
+    {
+      this.frame = frame;
+      dataRegion = frame.region(RowBasedFrameWriter.ROW_DATA_REGION);
+
+      this.coach = new FieldPositionHelper(
+          frame,
+          frame.region(RowBasedFrameWriter.ROW_OFFSET_REGION),
+          dataRegion,
+          columnIndex,
+          numFields
+      );
+    }
+
+    @Nonnull
+    @Override
+    public ColumnAccessor toAccessor()
+    {
+      return new ColumnAccessor()
+      {
+        @Override
+        public ColumnType getType()
+        {
+          return ColumnType.DOUBLE;
+        }
+
+        @Override
+        public int numRows()
+        {
+          return frame.numRows();
+        }
+
+        @Override
+        public boolean isNull(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+          return dataRegion.getByte(fieldPosition) == getNullIndicatorByte();
+        }
+
+        @Nullable
+        @Override
+        public Object getObject(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+
+          if (dataRegion.getByte(fieldPosition) == getNullIndicatorByte()) {
+            return null;
+          } else {
+            return getDoubleAtPosition(fieldPosition);
+          }
+        }
+
+        @Override
+        public double getDouble(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+
+          if (dataRegion.getByte(fieldPosition) == getNullIndicatorByte()) {
+            return 0L;
+          } else {
+            return getDoubleAtPosition(fieldPosition);
+          }
+        }
+
+        @Override
+        public float getFloat(int rowNum)
+        {
+          return (float) getDouble(rowNum);
+        }
+
+        @Override
+        public long getLong(int rowNum)
+        {
+          return (long) getDouble(rowNum);
+        }
+
+        @Override
+        public int getInt(int rowNum)
+        {
+          return (int) getDouble(rowNum);
+        }
+
+        @Override
+        public int compareRows(int lhsRowNum, int rhsRowNum)
+        {
+          long lhsPosition = coach.computeFieldPosition(lhsRowNum);
+          long rhsPosition = coach.computeFieldPosition(rhsRowNum);
+
+          final byte nullIndicatorByte = getNullIndicatorByte();
+          if (dataRegion.getByte(lhsPosition) == nullIndicatorByte) {
+            if (dataRegion.getByte(rhsPosition) == nullIndicatorByte) {
+              return 0;
+            } else {
+              return -1;
+            }
+          } else {
+            if (dataRegion.getByte(rhsPosition) == nullIndicatorByte) {
+              return 1;
+            } else {
+              return Double.compare(getDoubleAtPosition(lhsPosition), getDoubleAtPosition(rhsPosition));
+            }
+          }
+        }
+
+        private double getDoubleAtPosition(long lhsPosition)
+        {
+          return TransformUtils.detransformToDouble(dataRegion.getLong(lhsPosition + Byte.BYTES));
+        }
+      };
+    }
+
+    @Nullable
+    @Override
+    public <T> T as(Class<? extends T> clazz)
+    {
+      return null;
+    }
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/FieldPositionHelper.java b/processing/src/main/java/org/apache/druid/frame/field/FieldPositionHelper.java
new file mode 100644
index 00000000000..d4abe5300b4
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/frame/field/FieldPositionHelper.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.field;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.druid.frame.Frame;
+
+/**
+ * Helps compute the field position for a frame from the different regions in the frame.
+ */
+public class FieldPositionHelper
+{
+  private final Frame frame;
+  private final Memory offsetRegion;
+  private final Memory dataRegion;
+  private final int columnIndex;
+  private final long fieldsBytesSize;
+
+  public FieldPositionHelper(
+      Frame frame,
+      Memory offsetRegion,
+      Memory dataRegion,
+      int columnIndex,
+      int numFields
+  )
+  {
+    this.frame = frame;
+    this.offsetRegion = offsetRegion;
+    this.dataRegion = dataRegion;
+    this.columnIndex = columnIndex;
+    this.fieldsBytesSize = this.columnIndex == 0
+                           ? ((long) numFields) * Integer.BYTES
+                           : ((long) (this.columnIndex - 1)) * Integer.BYTES;
+  }
+
+  public long computeFieldPosition(int rowNum)
+  {
+    rowNum = frame.physicalRow(rowNum);
+    final long rowPosition = rowNum == 0 ? 0 : offsetRegion.getLong(((long) rowNum - 1) * Long.BYTES);
+    final long fieldPosition;
+    if (columnIndex == 0) {
+      fieldPosition = rowPosition + fieldsBytesSize;
+    } else {
+      fieldPosition = rowPosition + dataRegion.getInt(rowPosition + fieldsBytesSize);
+    }
+    return fieldPosition;
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/frame/field/FieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/FieldReader.java
index bc9d631361f..8cf0378071d 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/FieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/FieldReader.java
@@ -20,24 +20,32 @@
 package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.Memory;
+import org.apache.druid.frame.Frame;
 import org.apache.druid.frame.key.RowKey;
 import org.apache.druid.frame.key.RowKeyReader;
 import org.apache.druid.query.extraction.ExtractionFn;
+import org.apache.druid.query.rowsandcols.column.Column;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.column.RowSignature;
 
 import javax.annotation.Nullable;
 
 /**
  * Embeds the logic to read a specific field from row-based frames or from {@link RowKey}.
- *
+ * <p>
  * Most callers should use {@link org.apache.druid.frame.read.FrameReader} or
  * {@link RowKeyReader} rather than using this interface directly.
- *
+ * <p>
  * Stateless and immutable.
  */
 public interface FieldReader
 {
+  /**
+   * Create a {@link Column} which provides accses to the rows in the frame, via the {@link Column#toAccessor()}.
+   */
+  Column makeRACColumn(Frame frame, RowSignature signature, String columnName);
+
   /**
    * Create a {@link ColumnValueSelector} backed by some memory and a moveable pointer.
    */
diff --git a/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java
index e97af071824..7252265ba8a 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java
@@ -20,7 +20,11 @@
 package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.Memory;
+import org.apache.druid.error.NotYetImplemented;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.query.rowsandcols.column.Column;
 import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.column.RowSignature;
 
 import javax.annotation.Nullable;
 
@@ -60,4 +64,10 @@ public class FloatArrayFieldReader extends NumericArrayFieldReader
       }
     };
   }
+
+  @Override
+  public Column makeRACColumn(Frame frame, RowSignature signature, String columnName)
+  {
+    throw NotYetImplemented.ex(null, "Class cannot create an RAC column.");
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/FloatFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/FloatFieldReader.java
index 6617d563d67..3fc7213c73e 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/FloatFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/FloatFieldReader.java
@@ -21,11 +21,20 @@ package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.Memory;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.write.RowBasedFrameWriter;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.query.rowsandcols.column.Column;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.FloatColumnSelector;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 /**
  * Reads values written by {@link FloatFieldWriter}.
  *
@@ -99,4 +108,135 @@ public class FloatFieldReader extends NumericFieldReader
       return super._isNull();
     }
   }
+
+  @Override
+  public Column makeRACColumn(Frame frame, RowSignature signature, String columnName)
+  {
+    return new FloatFieldReaderColumn(frame, signature.indexOf(columnName), signature.size());
+  }
+
+  private class FloatFieldReaderColumn implements Column
+  {
+    private final Frame frame;
+    private final Memory dataRegion;
+    private final FieldPositionHelper coach;
+
+    public FloatFieldReaderColumn(Frame frame, int columnIndex, int numFields)
+    {
+      this.frame = frame;
+      dataRegion = frame.region(RowBasedFrameWriter.ROW_DATA_REGION);
+
+      this.coach = new FieldPositionHelper(
+          frame,
+          frame.region(RowBasedFrameWriter.ROW_OFFSET_REGION),
+          dataRegion,
+          columnIndex,
+          numFields
+      );
+    }
+
+    @Nonnull
+    @Override
+    public ColumnAccessor toAccessor()
+    {
+      return new ColumnAccessor()
+      {
+        @Override
+        public ColumnType getType()
+        {
+          return ColumnType.FLOAT;
+        }
+
+        @Override
+        public int numRows()
+        {
+          return frame.numRows();
+        }
+
+        @Override
+        public boolean isNull(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+          return dataRegion.getByte(fieldPosition) == getNullIndicatorByte();
+        }
+
+        @Nullable
+        @Override
+        public Object getObject(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+
+          if (dataRegion.getByte(fieldPosition) == getNullIndicatorByte()) {
+            return null;
+          } else {
+            return getFloatAtPosition(fieldPosition);
+          }
+        }
+
+        @Override
+        public double getDouble(int rowNum)
+        {
+          return getFloat(rowNum);
+        }
+
+        @Override
+        public float getFloat(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+
+          if (dataRegion.getByte(fieldPosition) == getNullIndicatorByte()) {
+            return 0L;
+          } else {
+            return getFloatAtPosition(fieldPosition);
+          }
+        }
+
+        @Override
+        public long getLong(int rowNum)
+        {
+          return (long) getFloat(rowNum);
+        }
+
+        @Override
+        public int getInt(int rowNum)
+        {
+          return (int) getFloat(rowNum);
+        }
+
+        @Override
+        public int compareRows(int lhsRowNum, int rhsRowNum)
+        {
+          long lhsPosition = coach.computeFieldPosition(lhsRowNum);
+          long rhsPosition = coach.computeFieldPosition(rhsRowNum);
+
+          final byte nullIndicatorByte = getNullIndicatorByte();
+          if (dataRegion.getByte(lhsPosition) == nullIndicatorByte) {
+            if (dataRegion.getByte(rhsPosition) == nullIndicatorByte) {
+              return 0;
+            } else {
+              return -1;
+            }
+          } else {
+            if (dataRegion.getByte(rhsPosition) == nullIndicatorByte) {
+              return 1;
+            } else {
+              return Float.compare(getFloatAtPosition(lhsPosition), getFloatAtPosition(rhsPosition));
+            }
+          }
+        }
+
+        private float getFloatAtPosition(long rhsPosition)
+        {
+          return TransformUtils.detransformToFloat(dataRegion.getInt(rhsPosition + Byte.BYTES));
+        }
+      };
+    }
+
+    @Nullable
+    @Override
+    public <T> T as(Class<? extends T> clazz)
+    {
+      return null;
+    }
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java
index 8f7578c07d3..ee77223aefe 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java
@@ -20,7 +20,11 @@
 package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.Memory;
+import org.apache.druid.error.NotYetImplemented;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.query.rowsandcols.column.Column;
 import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.column.RowSignature;
 
 import javax.annotation.Nullable;
 
@@ -60,4 +64,10 @@ public class LongArrayFieldReader extends NumericArrayFieldReader
       }
     };
   }
+
+  @Override
+  public Column makeRACColumn(Frame frame, RowSignature signature, String columnName)
+  {
+    throw NotYetImplemented.ex(null, "Class cannot create an RAC column.");
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/LongFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/LongFieldReader.java
index 8f3bbbf0451..9b514c93087 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/LongFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/LongFieldReader.java
@@ -21,11 +21,20 @@ package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.Memory;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.write.RowBasedFrameWriter;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.query.rowsandcols.column.Column;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.LongColumnSelector;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 /**
  * Reads values written by {@link LongFieldWriter}.
  *
@@ -68,7 +77,6 @@ public class LongFieldReader extends NumericFieldReader
 
   private static class LongFieldSelector extends NumericFieldReader.Selector implements LongColumnSelector
   {
-
     final Memory dataRegion;
     final ReadableFieldPointer fieldPointer;
 
@@ -99,4 +107,135 @@ public class LongFieldReader extends NumericFieldReader
       return super._isNull();
     }
   }
+
+  @Override
+  public Column makeRACColumn(Frame frame, RowSignature signature, String columnName)
+  {
+    return new LongFieldReaderColumn(frame, signature.indexOf(columnName), signature.size());
+  }
+
+  private class LongFieldReaderColumn implements Column
+  {
+    private final Frame frame;
+    private final Memory dataRegion;
+    private final FieldPositionHelper coach;
+
+    public LongFieldReaderColumn(Frame frame, int columnIndex, int numFields)
+    {
+      this.frame = frame;
+      this.dataRegion = frame.region(RowBasedFrameWriter.ROW_DATA_REGION);
+
+      this.coach = new FieldPositionHelper(
+          frame,
+          frame.region(RowBasedFrameWriter.ROW_OFFSET_REGION),
+          dataRegion,
+          columnIndex,
+          numFields
+      );
+    }
+
+    @Nonnull
+    @Override
+    public ColumnAccessor toAccessor()
+    {
+      return new ColumnAccessor()
+      {
+        @Override
+        public ColumnType getType()
+        {
+          return ColumnType.LONG;
+        }
+
+        @Override
+        public int numRows()
+        {
+          return frame.numRows();
+        }
+
+        @Override
+        public boolean isNull(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+          return dataRegion.getByte(fieldPosition) == getNullIndicatorByte();
+        }
+
+        @Nullable
+        @Override
+        public Object getObject(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+
+          if (dataRegion.getByte(fieldPosition) == getNullIndicatorByte()) {
+            return null;
+          } else {
+            return getLongAtPosition(fieldPosition);
+          }
+        }
+
+        @Override
+        public double getDouble(int rowNum)
+        {
+          return getLong(rowNum);
+        }
+
+        @Override
+        public float getFloat(int rowNum)
+        {
+          return getLong(rowNum);
+        }
+
+        @Override
+        public long getLong(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+
+          if (dataRegion.getByte(fieldPosition) == getNullIndicatorByte()) {
+            return 0L;
+          } else {
+            return getLongAtPosition(fieldPosition);
+          }
+        }
+
+        @Override
+        public int getInt(int rowNum)
+        {
+          return (int) getLong(rowNum);
+        }
+
+        @Override
+        public int compareRows(int lhsRowNum, int rhsRowNum)
+        {
+          long lhsPosition = coach.computeFieldPosition(lhsRowNum);
+          long rhsPosition = coach.computeFieldPosition(rhsRowNum);
+
+          final byte nullIndicatorByte = getNullIndicatorByte();
+          if (dataRegion.getByte(lhsPosition) == nullIndicatorByte) {
+            if (dataRegion.getByte(rhsPosition) == nullIndicatorByte) {
+              return 0;
+            } else {
+              return -1;
+            }
+          } else {
+            if (dataRegion.getByte(rhsPosition) == nullIndicatorByte) {
+              return 1;
+            } else {
+              return Long.compare(getLongAtPosition(lhsPosition), getLongAtPosition(rhsPosition));
+            }
+          }
+        }
+
+        private long getLongAtPosition(long rhsPosition)
+        {
+          return TransformUtils.detransformToLong(dataRegion.getLong(rhsPosition + Byte.BYTES));
+        }
+      };
+    }
+
+    @Nullable
+    @Override
+    public <T> T as(Class<? extends T> clazz)
+    {
+      return null;
+    }
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldReader.java
index 8d6f3958b1e..92dfbc98596 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldReader.java
@@ -29,7 +29,7 @@ import javax.annotation.Nullable;
 /**
  * Reader class for the fields written by {@link NumericArrayFieldWriter}. See the Javadoc for the writer for more
  * information on the format
- *
+ * <p>
  * The numeric array fields are byte comparable
  */
 public abstract class NumericArrayFieldReader implements FieldReader
diff --git a/processing/src/main/java/org/apache/druid/frame/field/NumericFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/NumericFieldReader.java
index 1e11cfa65f3..bb0cc4faed6 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/NumericFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/NumericFieldReader.java
@@ -50,6 +50,11 @@ public abstract class NumericFieldReader implements FieldReader
     }
   }
 
+  public byte getNullIndicatorByte()
+  {
+    return nullIndicatorByte;
+  }
+
   @Override
   public ColumnValueSelector<?> makeColumnValueSelector(Memory memory, ReadableFieldPointer fieldPointer)
   {
@@ -94,7 +99,7 @@ public abstract class NumericFieldReader implements FieldReader
   /**
    * Helper class which allows the inheritors to fetch the nullity of the field located at fieldPointer's position in
    * the dataRegion.
-   *
+   * <p>
    * The implementations of the column value selectors returned by the {@link #getColumnValueSelector} can inherit this
    * class and call {@link #_isNull()} in their {@link ColumnValueSelector#isNull()} to offload the responsibility of
    * detecting null elements to this Selector, instead of reworking the null handling
diff --git a/processing/src/main/java/org/apache/druid/frame/field/StringFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/StringFieldReader.java
index 2513a2d2444..1c51a914e0d 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/StringFieldReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/StringFieldReader.java
@@ -23,51 +23,64 @@ import com.google.common.primitives.Ints;
 import it.unimi.dsi.fastutil.objects.ObjectArrays;
 import org.apache.datasketches.memory.Memory;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.error.NotYetImplemented;
+import org.apache.druid.frame.Frame;
 import org.apache.druid.frame.read.FrameReaderUtils;
+import org.apache.druid.frame.write.RowBasedFrameWriter;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.query.extraction.ExtractionFn;
 import org.apache.druid.query.filter.DruidPredicateFactory;
 import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.query.rowsandcols.column.Column;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
+import org.apache.druid.query.rowsandcols.column.accessor.ObjectColumnAccessorBase;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.DimensionSelectorUtils;
 import org.apache.druid.segment.IdLookup;
 import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.data.IndexedInts;
 import org.apache.druid.segment.data.RangeIndexedInts;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 
 /**
  * Reads fields written by {@link StringFieldWriter} or {@link StringArrayFieldWriter}.
- *
+ * <p>
  * Strings are written in UTF8 and terminated by {@link StringFieldWriter#VALUE_TERMINATOR}. Note that this byte
  * appears in valid UTF8 encodings if and only if the string contains a NUL (char 0). Therefore, this field writer
  * cannot write out strings containing NUL characters.
- *
+ * <p>
  * All rows are terminated by {@link StringFieldWriter#ROW_TERMINATOR}.
- *
+ * <p>
  * Empty rows are represented in one byte: solely that {@link StringFieldWriter#ROW_TERMINATOR}. Rows that are null
  * themselves (i.e., a null array) are represented as a {@link StringFieldWriter#NULL_ROW} followed by a
  * {@link StringFieldWriter#ROW_TERMINATOR}. This encoding for null arrays is decoded by older readers as an
  * empty array; null arrays are a feature that did not exist in earlier versions of the code.
- *
+ * <p>
  * Null strings are stored as {@link StringFieldWriter#NULL_BYTE}. All other strings are prepended by
  * {@link StringFieldWriter#NOT_NULL_BYTE} byte to differentiate them from nulls.
- *
+ * <p>
  * This encoding allows the encoded data to be compared as bytes in a way that matches the behavior of
  * {@link org.apache.druid.segment.StringDimensionHandler#DIMENSION_SELECTOR_COMPARATOR}, except null and
  * empty list are not considered equal.
  */
 public class StringFieldReader implements FieldReader
 {
+  public static final byte[] EXPECTED_BYTES_FOR_NULL = {
+      StringFieldWriter.NULL_BYTE, StringFieldWriter.VALUE_TERMINATOR, StringFieldWriter.ROW_TERMINATOR
+  };
   private final boolean asArray;
 
   public StringFieldReader()
@@ -123,6 +136,16 @@ public class StringFieldReader implements FieldReader
     }
   }
 
+  @Override
+  public Column makeRACColumn(Frame frame, RowSignature signature, String columnName)
+  {
+    if (asArray) {
+      return new StringArrayFieldReaderColumn(frame, signature.indexOf(columnName), signature.size());
+    } else {
+      return new StringFieldReaderColumn(frame, signature.indexOf(columnName), signature.size());
+    }
+  }
+
   /**
    * Selector that reads a value from a location pointed to by {@link ReadableFieldPointer}.
    */
@@ -297,70 +320,296 @@ public class StringFieldReader implements FieldReader
     {
       currentUtf8StringsIsNull = false;
       currentUtf8Strings.clear();
-
-      long position = fieldPosition;
-      long limit = memory.getCapacity();
-
-      boolean rowTerminatorSeen = false;
-
-      while (position < limit && !rowTerminatorSeen) {
-        final byte kind = memory.getByte(position);
-        position++;
-
-        switch (kind) {
-          case StringFieldWriter.VALUE_TERMINATOR: // Or NULL_ROW (same byte value)
-            if (position == fieldPosition + 1) {
-              // It was NULL_ROW.
-              currentUtf8StringsIsNull = true;
-            }
-
-            // Skip; next byte will be a null/not-null byte or a row terminator.
-            break;
-
-          case StringFieldWriter.ROW_TERMINATOR:
-            // Skip; this is the end of the row, so we'll fall through to the return statement.
-            rowTerminatorSeen = true;
-            break;
-
-          case StringFieldWriter.NULL_BYTE:
-            currentUtf8Strings.add(null);
-            break;
-
-          case StringFieldWriter.NOT_NULL_BYTE:
-            for (long i = position; ; i++) {
-              if (i >= limit) {
-                throw new ISE("Value overrun");
-              }
-
-              final byte b = memory.getByte(i);
-
-              if (b == StringFieldWriter.VALUE_TERMINATOR) {
-                final int len = Ints.checkedCast(i - position);
-
-                if (len == 0 && NullHandling.replaceWithDefault()) {
-                  // Empty strings and nulls are the same in this mode.
-                  currentUtf8Strings.add(null);
-                } else {
-                  final ByteBuffer buf = FrameReaderUtils.readByteBuffer(memory, position, len);
-                  currentUtf8Strings.add(buf);
-                }
-
-                position += len;
-
-                break;
-              }
-            }
-
-            break;
-
-          default:
-            throw new ISE("Invalid value start byte [%s]", kind);
-        }
-      }
-
-      if (!rowTerminatorSeen) {
-        throw new ISE("Unexpected end of field");
-      }
+      currentUtf8StringsIsNull = addStringsToList(memory, fieldPosition, currentUtf8Strings);
     }
   }
+
+  private static class StringFieldReaderColumn implements Column
+  {
+    private final Frame frame;
+    private final Memory dataRegion;
+    private final FieldPositionHelper coach;
+
+    public StringFieldReaderColumn(Frame frame, int columnIndex, int numFields)
+    {
+      this.frame = frame;
+      this.dataRegion = frame.region(RowBasedFrameWriter.ROW_DATA_REGION);
+
+      this.coach = new FieldPositionHelper(
+          frame,
+          frame.region(RowBasedFrameWriter.ROW_OFFSET_REGION),
+          dataRegion,
+          columnIndex,
+          numFields
+      );
+    }
+
+    @Nonnull
+    @Override
+    public ColumnAccessor toAccessor()
+    {
+      return new ObjectColumnAccessorBase()
+      {
+        @Override
+        public ColumnType getType()
+        {
+          return ColumnType.STRING;
+        }
+
+        @Override
+        public int numRows()
+        {
+          return frame.numRows();
+        }
+
+        @Override
+        public boolean isNull(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+          byte[] nullBytes = new byte[3];
+          dataRegion.getByteArray(fieldPosition, nullBytes, 0, 3);
+          return Arrays.equals(nullBytes, EXPECTED_BYTES_FOR_NULL);
+        }
+
+        @Override
+        public int compareRows(int lhsRowNum, int rhsRowNum)
+        {
+          ByteBuffer lhs = getUTF8BytesAtPosition(coach.computeFieldPosition(lhsRowNum));
+          ByteBuffer rhs = getUTF8BytesAtPosition(coach.computeFieldPosition(rhsRowNum));
+
+          if (lhs == null) {
+            if (rhs == null) {
+              return 0;
+            } else {
+              return -1;
+            }
+          } else {
+            if (rhs == null) {
+              return 1;
+            } else {
+              return lhs.compareTo(rhs);
+            }
+          }
+        }
+
+        @Override
+        protected Object getVal(int rowNum)
+        {
+          return getStringAtPosition(coach.computeFieldPosition(rowNum));
+        }
+
+        @Override
+        protected Comparator<Object> getComparator()
+        {
+          // we implement compareRows and thus don't need to actually implement this method
+          throw new UnsupportedOperationException();
+        }
+
+        @Nullable
+        private String getStringAtPosition(long fieldPosition)
+        {
+          return StringUtils.fromUtf8Nullable(getUTF8BytesAtPosition(fieldPosition));
+        }
+
+        @Nullable
+        private ByteBuffer getUTF8BytesAtPosition(long fieldPosition)
+        {
+          ArrayList<ByteBuffer> buffers = new ArrayList<>();
+          final boolean isNull = addStringsToList(dataRegion, fieldPosition, buffers);
+          if (isNull) {
+            return null;
+          } else {
+            if (buffers.size() > 1) {
+              throw DruidException.defensive(
+                  "Can only work with single-valued strings, should use a COMPLEX or ARRAY typed Column instead"
+              );
+            }
+            return buffers.get(0);
+          }
+        }
+      };
+    }
+
+    @Nullable
+    @Override
+    public <T> T as(Class<? extends T> clazz)
+    {
+      return null;
+    }
+  }
+
+  private static class StringArrayFieldReaderColumn implements Column
+  {
+    private final Frame frame;
+    private final Memory dataRegion;
+    private final FieldPositionHelper coach;
+
+    public StringArrayFieldReaderColumn(Frame frame, int columnIndex, int numFields)
+    {
+      this.frame = frame;
+      this.dataRegion = frame.region(RowBasedFrameWriter.ROW_DATA_REGION);
+
+      this.coach = new FieldPositionHelper(
+          frame,
+          frame.region(RowBasedFrameWriter.ROW_OFFSET_REGION),
+          this.dataRegion,
+          columnIndex,
+          numFields
+      );
+    }
+
+    @Nonnull
+    @Override
+    public ColumnAccessor toAccessor()
+    {
+      return new ObjectColumnAccessorBase()
+      {
+        @Override
+        public ColumnType getType()
+        {
+          return ColumnType.STRING_ARRAY;
+        }
+
+        @Override
+        public int numRows()
+        {
+          return frame.numRows();
+        }
+
+        @Override
+        public boolean isNull(int rowNum)
+        {
+          final long fieldPosition = coach.computeFieldPosition(rowNum);
+          byte[] nullBytes = new byte[3];
+          dataRegion.getByteArray(fieldPosition, nullBytes, 0, 3);
+          return Arrays.equals(nullBytes, EXPECTED_BYTES_FOR_NULL);
+        }
+
+        @Override
+        public int compareRows(int lhsRowNum, int rhsRowNum)
+        {
+          throw NotYetImplemented.ex(
+              null,
+              "Should implement this by comparing the actual bytes in the frame, they should be comparable"
+          );
+        }
+
+        @Override
+        protected Object getVal(int rowNum)
+        {
+          return getStringsAtPosition(coach.computeFieldPosition(rowNum));
+        }
+
+        @Override
+        protected Comparator<Object> getComparator()
+        {
+          // we implement compareRows and thus don't need to actually implement this method
+          throw new UnsupportedOperationException();
+        }
+
+        @Nullable
+        private List<String> getStringsAtPosition(long fieldPosition)
+        {
+          final List<ByteBuffer> bufs = getUTF8BytesAtPosition(fieldPosition);
+          if (bufs == null) {
+            return null;
+          }
+
+          final ArrayList<String> retVal = new ArrayList<>(bufs.size());
+          for (ByteBuffer buf : bufs) {
+            retVal.add(StringUtils.fromUtf8Nullable(buf));
+          }
+          return retVal;
+        }
+
+        @Nullable
+        private List<ByteBuffer> getUTF8BytesAtPosition(long fieldPosition)
+        {
+          ArrayList<ByteBuffer> buffers = new ArrayList<>();
+          final boolean isNull = addStringsToList(dataRegion, fieldPosition, buffers);
+          if (isNull) {
+            return null;
+          } else {
+            return buffers;
+          }
+        }
+      };
+    }
+
+    @Nullable
+    @Override
+    public <T> T as(Class<? extends T> clazz)
+    {
+      return null;
+    }
+  }
+
+  private static boolean addStringsToList(Memory memory, long fieldPosition, List<ByteBuffer> list)
+  {
+    long position = fieldPosition;
+    long limit = memory.getCapacity();
+
+    boolean rowTerminatorSeen = false;
+    boolean isEffectivelyNull = false;
+
+    while (position < limit && !rowTerminatorSeen) {
+      final byte kind = memory.getByte(position);
+      position++;
+
+      switch (kind) {
+        case StringFieldWriter.VALUE_TERMINATOR: // Or NULL_ROW (same byte value)
+          if (position == fieldPosition + 1) {
+            // It was NULL_ROW.
+            isEffectivelyNull = true;
+          }
+
+          // Skip; next byte will be a null/not-null byte or a row terminator.
+          break;
+
+        case StringFieldWriter.ROW_TERMINATOR:
+          // Skip; this is the end of the row, so we'll fall through to the return statement.
+          rowTerminatorSeen = true;
+          break;
+
+        case StringFieldWriter.NULL_BYTE:
+          list.add(null);
+          break;
+
+        case StringFieldWriter.NOT_NULL_BYTE:
+          for (long i = position; ; i++) {
+            if (i >= limit) {
+              throw new ISE("Value overrun");
+            }
+
+            final byte b = memory.getByte(i);
+
+            if (b == StringFieldWriter.VALUE_TERMINATOR) {
+              final int len = Ints.checkedCast(i - position);
+
+              if (len == 0 && NullHandling.replaceWithDefault()) {
+                // Empty strings and nulls are the same in this mode.
+                list.add(null);
+              } else {
+                final ByteBuffer buf = FrameReaderUtils.readByteBuffer(memory, position, len);
+                list.add(buf);
+              }
+
+              position += len;
+
+              break;
+            }
+          }
+
+          break;
+
+        default:
+          throw new ISE("Invalid value start byte [%s]", kind);
+      }
+    }
+
+    if (!rowTerminatorSeen) {
+      throw new ISE("Unexpected end of field");
+    }
+    return isEffectivelyNull;
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/concrete/RowBasedFrameRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/concrete/RowBasedFrameRowsAndColumns.java
index 234410bc070..fa17984e9ba 100644
--- a/processing/src/main/java/org/apache/druid/query/rowsandcols/concrete/RowBasedFrameRowsAndColumns.java
+++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/concrete/RowBasedFrameRowsAndColumns.java
@@ -19,12 +19,13 @@
 
 package org.apache.druid.query.rowsandcols.concrete;
 
+import org.apache.druid.error.DruidException;
 import org.apache.druid.frame.Frame;
 import org.apache.druid.frame.FrameType;
+import org.apache.druid.frame.field.FieldReader;
+import org.apache.druid.frame.field.FieldReaders;
 import org.apache.druid.frame.read.FrameReader;
-import org.apache.druid.frame.read.columnar.FrameColumnReaders;
 import org.apache.druid.frame.segment.FrameStorageAdapter;
-import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.query.rowsandcols.RowsAndColumns;
 import org.apache.druid.query.rowsandcols.column.Column;
@@ -65,6 +66,7 @@ public class RowBasedFrameRowsAndColumns implements RowsAndColumns, AutoCloseabl
   @Override
   public Column findColumn(String name)
   {
+    // Use contains so that we can negative cache.
     if (!colCache.containsKey(name)) {
       final int columnIndex = signature.indexOf(name);
       if (columnIndex < 0) {
@@ -72,9 +74,16 @@ public class RowBasedFrameRowsAndColumns implements RowsAndColumns, AutoCloseabl
       } else {
         final ColumnType columnType = signature
             .getColumnType(columnIndex)
-            .orElseThrow(() -> new ISE("just got the id, why is columnType not there?"));
+            .orElseThrow(
+                () -> DruidException.defensive(
+                    "just got the id [%s][%s], why is columnType not there?",
+                    columnIndex,
+                    name
+                )
+            );
 
-        colCache.put(name, FrameColumnReaders.create(name, columnIndex, columnType).readRACColumn(frame));
+        final FieldReader reader = FieldReaders.create(name, columnType);
+        colCache.put(name, reader.makeRACColumn(frame, signature, name));
       }
     }
     return colCache.get(name);
diff --git a/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java
index 00fafe53f62..175e9de2a56 100644
--- a/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java
@@ -21,14 +21,20 @@ package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.frame.key.KeyOrder;
+import org.apache.druid.frame.write.FrameWriterTestData;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.extraction.SubstringDimExtractionFn;
 import org.apache.druid.query.filter.DruidObjectPredicate;
 import org.apache.druid.query.filter.StringPredicateDruidPredicateFactory;
+import org.apache.druid.query.rowsandcols.MapOfColumnsRowsAndColumns;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
+import org.apache.druid.query.rowsandcols.concrete.RowBasedFrameRowsAndColumnsTest;
 import org.apache.druid.segment.BaseDoubleColumnValueSelector;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.IndexedInts;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.After;
@@ -42,6 +48,9 @@ import org.mockito.junit.MockitoJUnit;
 import org.mockito.junit.MockitoRule;
 import org.mockito.quality.Strictness;
 
+import java.util.List;
+import java.util.Objects;
+
 public class DoubleFieldReaderTest extends InitializedNullHandlingTest
 {
   private static final long MEMORY_POSITION = 1;
@@ -143,6 +152,27 @@ public class DoubleFieldReaderTest extends InitializedNullHandlingTest
     }
   }
 
+  @Test
+  public void testCompareRows()
+  {
+    final List<Double> rows = FrameWriterTestData.TEST_DOUBLES.getData(KeyOrder.ASCENDING);
+
+    final ColumnAccessor accessor =
+        RowBasedFrameRowsAndColumnsTest.MAKER.apply(
+            MapOfColumnsRowsAndColumns.builder()
+                                      .add("dim1", rows.toArray(), ColumnType.DOUBLE)
+                                      .build()
+        ).findColumn("dim1").toAccessor();
+
+    for (int i = 1; i < rows.size(); i++) {
+      if (Objects.equals(accessor.getObject(i - 1), accessor.getObject(i))) {
+        Assert.assertEquals(0, accessor.compareRows(i - 1, i));
+      } else {
+        Assert.assertTrue(accessor.compareRows(i - 1, i) < 0);
+      }
+    }
+  }
+
   @Test
   public void test_makeDimensionSelector_aValue()
   {
diff --git a/processing/src/test/java/org/apache/druid/frame/field/FieldReaderRACTest.java b/processing/src/test/java/org/apache/druid/frame/field/FieldReaderRACTest.java
new file mode 100644
index 00000000000..70ff7037b8c
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/frame/field/FieldReaderRACTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.field;
+
+import org.apache.druid.error.DruidExceptionMatcher;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.FrameType;
+import org.apache.druid.frame.testutil.FrameTestUtil;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
+import org.apache.druid.query.rowsandcols.concrete.RowBasedFrameRowsAndColumns;
+import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.SimpleAscendingOffset;
+import org.apache.druid.segment.TestIndex;
+import org.apache.druid.segment.column.BaseColumn;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class FieldReaderRACTest extends InitializedNullHandlingTest
+{
+  final DruidExceptionMatcher noArraysMatcher = DruidExceptionMatcher
+      .defensive()
+      .expectMessageIs("Can only work with single-valued strings, should use a COMPLEX or ARRAY typed Column instead");
+
+  @Test
+  public void testDataSet() throws IOException
+  {
+    final QueryableIndex index = TestIndex.getMMappedTestIndex();
+    final QueryableIndexStorageAdapter storageAdapter = new QueryableIndexStorageAdapter(index);
+    final Frame frame = FrameTestUtil.adapterToFrame(storageAdapter, FrameType.ROW_BASED);
+
+    final RowSignature siggy = storageAdapter.getRowSignature();
+    final RowBasedFrameRowsAndColumns rowBasedRAC = new RowBasedFrameRowsAndColumns(frame, siggy);
+
+    for (String columnName : siggy.getColumnNames()) {
+      final ColumnHolder colHolder = index.getColumnHolder(columnName);
+      final boolean multiValue = colHolder.getCapabilities().hasMultipleValues().isTrue();
+
+      try (BaseColumn col = colHolder.getColumn()) {
+        final ColumnAccessor racCol = rowBasedRAC.findColumn(columnName).toAccessor();
+
+        final SimpleAscendingOffset offset = new SimpleAscendingOffset(racCol.numRows());
+        final ColumnValueSelector<?> selector = col.makeColumnValueSelector(offset);
+        while (offset.withinBounds()) {
+          if (multiValue) {
+            noArraysMatcher.assertThrowsAndMatches(() -> racCol.getObject(offset.getOffset()));
+          } else {
+            final Object racObj = racCol.getObject(offset.getOffset());
+            Assert.assertEquals(racCol.isNull(offset.getOffset()), racCol.getObject(offset.getOffset()) == null);
+            Assert.assertEquals(selector.getObject(), racObj);
+          }
+          offset.increment();
+        }
+      }
+    }
+  }
+}
diff --git a/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java
index 4b8d10ca0e0..3669c89232d 100644
--- a/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java
@@ -21,14 +21,20 @@ package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.frame.key.KeyOrder;
+import org.apache.druid.frame.write.FrameWriterTestData;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.extraction.SubstringDimExtractionFn;
 import org.apache.druid.query.filter.DruidObjectPredicate;
 import org.apache.druid.query.filter.StringPredicateDruidPredicateFactory;
+import org.apache.druid.query.rowsandcols.MapOfColumnsRowsAndColumns;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
+import org.apache.druid.query.rowsandcols.concrete.RowBasedFrameRowsAndColumnsTest;
 import org.apache.druid.segment.BaseFloatColumnValueSelector;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.IndexedInts;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.After;
@@ -42,6 +48,9 @@ import org.mockito.junit.MockitoJUnit;
 import org.mockito.junit.MockitoRule;
 import org.mockito.quality.Strictness;
 
+import java.util.List;
+import java.util.Objects;
+
 public class FloatFieldReaderTest extends InitializedNullHandlingTest
 {
   private static final long MEMORY_POSITION = 1;
@@ -75,6 +84,27 @@ public class FloatFieldReaderTest extends InitializedNullHandlingTest
     Assert.assertEquals(NullHandling.sqlCompatible(), FloatFieldReader.forPrimitive().isNull(memory, MEMORY_POSITION));
   }
 
+  @Test
+  public void testCompareRows()
+  {
+    final List<Float> rows = FrameWriterTestData.TEST_FLOATS.getData(KeyOrder.ASCENDING);
+
+    final ColumnAccessor accessor =
+        RowBasedFrameRowsAndColumnsTest.MAKER.apply(
+            MapOfColumnsRowsAndColumns.builder()
+                                      .add("dim1", rows.toArray(), ColumnType.FLOAT)
+                                      .build()
+        ).findColumn("dim1").toAccessor();
+
+    for (int i = 1; i < rows.size(); i++) {
+      if (Objects.equals(accessor.getObject(i - 1), accessor.getObject(i))) {
+        Assert.assertEquals(0, accessor.compareRows(i - 1, i));
+      } else {
+        Assert.assertTrue(accessor.compareRows(i - 1, i) < 0);
+      }
+    }
+  }
+
   @Test
   public void test_isNull_aValue()
   {
diff --git a/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java
index aab55654e5a..266d86e1283 100644
--- a/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java
@@ -21,14 +21,20 @@ package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.frame.key.KeyOrder;
+import org.apache.druid.frame.write.FrameWriterTestData;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.extraction.SubstringDimExtractionFn;
 import org.apache.druid.query.filter.DruidObjectPredicate;
 import org.apache.druid.query.filter.StringPredicateDruidPredicateFactory;
+import org.apache.druid.query.rowsandcols.MapOfColumnsRowsAndColumns;
+import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
+import org.apache.druid.query.rowsandcols.concrete.RowBasedFrameRowsAndColumnsTest;
 import org.apache.druid.segment.BaseLongColumnValueSelector;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.IndexedInts;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.After;
@@ -42,6 +48,9 @@ import org.mockito.junit.MockitoJUnit;
 import org.mockito.junit.MockitoRule;
 import org.mockito.quality.Strictness;
 
+import java.util.List;
+import java.util.Objects;
+
 public class LongFieldReaderTest extends InitializedNullHandlingTest
 {
   private static final long MEMORY_POSITION = 1;
@@ -202,6 +211,28 @@ public class LongFieldReaderTest extends InitializedNullHandlingTest
     Assert.assertFalse(readSelector.makeValueMatcher(StringPredicateDruidPredicateFactory.equalTo("2")).matches(false));
   }
 
+
+  @Test
+  public void testCompareRows()
+  {
+    final List<Long> rows = FrameWriterTestData.TEST_LONGS.getData(KeyOrder.ASCENDING);
+
+    final ColumnAccessor accessor =
+        RowBasedFrameRowsAndColumnsTest.MAKER.apply(
+            MapOfColumnsRowsAndColumns.builder()
+                                      .add("dim1", rows.toArray(), ColumnType.LONG)
+                                      .build()
+        ).findColumn("dim1").toAccessor();
+
+    for (int i = 1; i < rows.size(); i++) {
+      if (Objects.equals(accessor.getObject(i - 1), accessor.getObject(i))) {
+        Assert.assertEquals(0, accessor.compareRows(i - 1, i));
+      } else {
+        Assert.assertTrue(accessor.compareRows(i - 1, i) < 0);
+      }
+    }
+  }
+
   private void writeToMemory(final Long value)
   {
     Mockito.when(writeSelector.isNull()).thenReturn(value == null);
diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/RowsAndColumnsTestBase.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/RowsAndColumnsTestBase.java
index 56be3d50f20..7b639b3d48d 100644
--- a/processing/src/test/java/org/apache/druid/query/rowsandcols/RowsAndColumnsTestBase.java
+++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/RowsAndColumnsTestBase.java
@@ -24,6 +24,8 @@ import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.rowsandcols.concrete.ColumnBasedFrameRowsAndColumns;
 import org.apache.druid.query.rowsandcols.concrete.ColumnBasedFrameRowsAndColumnsTest;
+import org.apache.druid.query.rowsandcols.concrete.RowBasedFrameRowsAndColumns;
+import org.apache.druid.query.rowsandcols.concrete.RowBasedFrameRowsAndColumnsTest;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -67,7 +69,8 @@ public abstract class RowsAndColumnsTestBase
         new Object[]{ConcatRowsAndColumns.class, ConcatRowsAndColumnsTest.MAKER},
         new Object[]{RearrangedRowsAndColumns.class, RearrangedRowsAndColumnsTest.MAKER},
         new Object[]{ColumnBasedFrameRowsAndColumns.class, ColumnBasedFrameRowsAndColumnsTest.MAKER},
-        new Object[]{StorageAdapterRowsAndColumns.class, StorageAdapterRowsAndColumnsTest.MAKER}
+        new Object[]{StorageAdapterRowsAndColumns.class, StorageAdapterRowsAndColumnsTest.MAKER},
+        new Object[]{RowBasedFrameRowsAndColumns.class, RowBasedFrameRowsAndColumnsTest.MAKER}
     );
   }
 
diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/concrete/ColumnBasedFrameRowsAndColumnsTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/concrete/ColumnBasedFrameRowsAndColumnsTest.java
index cd1bb1b81ec..acfcbe6f83e 100644
--- a/processing/src/test/java/org/apache/druid/query/rowsandcols/concrete/ColumnBasedFrameRowsAndColumnsTest.java
+++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/concrete/ColumnBasedFrameRowsAndColumnsTest.java
@@ -33,17 +33,13 @@ public class ColumnBasedFrameRowsAndColumnsTest extends RowsAndColumnsTestBase
     super(ColumnBasedFrameRowsAndColumns.class);
   }
 
-  public static Function<MapOfColumnsRowsAndColumns, ColumnBasedFrameRowsAndColumns> MAKER = input -> {
-
-    return buildFrame(input);
-  };
+  public static Function<MapOfColumnsRowsAndColumns, ColumnBasedFrameRowsAndColumns> MAKER = ColumnBasedFrameRowsAndColumnsTest::buildFrame;
 
   public static ColumnBasedFrameRowsAndColumns buildFrame(MapOfColumnsRowsAndColumns input)
   {
     LazilyDecoratedRowsAndColumns rac = new LazilyDecoratedRowsAndColumns(input, null, null, null, OffsetLimit.limit(Integer.MAX_VALUE), null, null);
 
     rac.numRows(); // materialize
-
     return (ColumnBasedFrameRowsAndColumns) rac.getBase();
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/concrete/RowBasedFrameRowsAndColumnsTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/concrete/RowBasedFrameRowsAndColumnsTest.java
new file mode 100644
index 00000000000..867e83d9e00
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/concrete/RowBasedFrameRowsAndColumnsTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.rowsandcols.concrete;
+
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.allocation.ArenaMemoryAllocatorFactory;
+import org.apache.druid.frame.write.FrameWriter;
+import org.apache.druid.frame.write.FrameWriters;
+import org.apache.druid.query.rowsandcols.MapOfColumnsRowsAndColumns;
+import org.apache.druid.query.rowsandcols.RowsAndColumnsTestBase;
+import org.apache.druid.query.rowsandcols.column.Column;
+import org.apache.druid.query.rowsandcols.semantic.ColumnSelectorFactoryMaker;
+import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.column.RowSignature;
+
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+
+public class RowBasedFrameRowsAndColumnsTest extends RowsAndColumnsTestBase
+{
+  public RowBasedFrameRowsAndColumnsTest()
+  {
+    super(RowBasedFrameRowsAndColumns.class);
+  }
+
+  public static Function<MapOfColumnsRowsAndColumns, RowBasedFrameRowsAndColumns> MAKER = RowBasedFrameRowsAndColumnsTest::buildFrame;
+
+  private static RowBasedFrameRowsAndColumns buildFrame(MapOfColumnsRowsAndColumns rac)
+  {
+    final AtomicInteger rowId = new AtomicInteger(0);
+    final int numRows = rac.numRows();
+    final ColumnSelectorFactoryMaker csfm = ColumnSelectorFactoryMaker.fromRAC(rac);
+    final ColumnSelectorFactory selectorFactory = csfm.make(rowId);
+
+    final RowSignature.Builder sigBob = RowSignature.builder();
+    final ArenaMemoryAllocatorFactory memFactory = new ArenaMemoryAllocatorFactory(200 << 20);
+
+
+    for (String column : rac.getColumnNames()) {
+      final Column racColumn = rac.findColumn(column);
+      if (racColumn == null) {
+        continue;
+      }
+      sigBob.add(column, racColumn.toAccessor().getType());
+    }
+
+    final RowSignature signature = sigBob.build();
+    final FrameWriter frameWriter = FrameWriters.makeRowBasedFrameWriterFactory(
+        memFactory,
+        signature,
+        Collections.emptyList(),
+        false
+    ).newFrameWriter(selectorFactory);
+
+    rowId.set(0);
+    for (; rowId.get() < numRows; rowId.incrementAndGet()) {
+      frameWriter.addSelection();
+    }
+
+    return new RowBasedFrameRowsAndColumns(Frame.wrap(frameWriter.toByteArray()), signature);
+  }
+}

From acadc2df20117a3c5aac836500ef0b11880990ea Mon Sep 17 00:00:00 2001
From: Abhishek Radhakrishnan <abhishek.rb19@gmail.com>
Date: Tue, 13 Aug 2024 07:50:03 -0700
Subject: [PATCH 10/99] Handle Delta StructType, ArrayType and MapType (#16884)

Handle the following Delta complex types:
a. StructType as JSON
b. ArrayType as Java list
c. MapType as Java map

Generate and add a new Delta table complex-types-table that contains the above complex types for testing.

Update the tests to include a parameterized test with complex-types-table, with the expectations defined in ComplexTypesDeltaTable.java.
---
 .../druid/delta/input/DeltaInputRow.java      |  14 ++
 .../delta/input/ComplexTypesDeltaTable.java   | 135 ++++++++++++++++++
 .../druid/delta/input/DeltaInputRowTest.java  |   5 +-
 .../delta/input/DeltaInputSourceTest.java     |   5 +
 .../src/test/resources/README.md              |  11 ++
 ...-9b74-0cc7b853103a-c000.snappy.parquet.crc | Bin 0 -> 20 bytes
 ...-834e-2a1cb6601eb8-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 ...-82a4-f3f273308e53-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 ...-bb38-f627c47eb20b-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 ...-bc77-0106e7f28f7a-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 ...-b605-506c942cd969-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 52 bytes
 .../_delta_log/00000000000000000000.json      |   8 ++
 ...4a53-9b74-0cc7b853103a-c000.snappy.parquet | Bin 0 -> 1488 bytes
 ...4e91-834e-2a1cb6601eb8-c000.snappy.parquet | Bin 0 -> 3288 bytes
 ...4ef3-82a4-f3f273308e53-c000.snappy.parquet | Bin 0 -> 3291 bytes
 ...404a-bb38-f627c47eb20b-c000.snappy.parquet | Bin 0 -> 3289 bytes
 ...4141-bc77-0106e7f28f7a-c000.snappy.parquet | Bin 0 -> 3290 bytes
 ...43fe-b605-506c942cd969-c000.snappy.parquet | Bin 0 -> 3291 bytes
 .../src/test/resources/create_delta_table.py  |  63 +++++++-
 20 files changed, 235 insertions(+), 6 deletions(-)
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet
 create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet

diff --git a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java
index acf909452a0..442412dd154 100644
--- a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java
+++ b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java
@@ -19,6 +19,10 @@
 
 package org.apache.druid.delta.input;
 
+import io.delta.kernel.data.ArrayValue;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.ArrayType;
 import io.delta.kernel.types.BinaryType;
 import io.delta.kernel.types.BooleanType;
 import io.delta.kernel.types.ByteType;
@@ -29,6 +33,7 @@ import io.delta.kernel.types.DoubleType;
 import io.delta.kernel.types.FloatType;
 import io.delta.kernel.types.IntegerType;
 import io.delta.kernel.types.LongType;
+import io.delta.kernel.types.MapType;
 import io.delta.kernel.types.ShortType;
 import io.delta.kernel.types.StringType;
 import io.delta.kernel.types.StructField;
@@ -197,6 +202,15 @@ public class DeltaInputRow implements InputRow
       return String.valueOf(charArray);
     } else if (dataType instanceof DecimalType) {
       return dataRow.getDecimal(columnOrdinal).longValue();
+    } else if (dataType instanceof StructType) {
+      final io.delta.kernel.data.Row structRow = dataRow.getStruct(columnOrdinal);
+      return RowSerde.convertRowToJsonObject(structRow);
+    } else if (dataType instanceof ArrayType) {
+      final ArrayValue arrayRow = dataRow.getArray(columnOrdinal);
+      return VectorUtils.toJavaList(arrayRow);
+    } else if (dataType instanceof MapType) {
+      final MapValue map = dataRow.getMap(columnOrdinal);
+      return VectorUtils.toJavaMap(map);
     } else {
       throw InvalidInput.exception(
           "Unsupported data type[%s] for fieldName[%s].",
diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java
new file mode 100644
index 00000000000..7fdffc03041
--- /dev/null
+++ b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.delta.input;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.data.input.ColumnsFilter;
+import org.apache.druid.data.input.InputRowSchema;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.segment.AutoTypeColumnSchema;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Refer to extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md to generate the
+ * sample complex types Delta Lake table used in the unit tests.
+ *
+ */
+public class ComplexTypesDeltaTable
+{
+  /**
+   * The Delta table path used by unit tests.
+   */
+  public static final String DELTA_TABLE_PATH = "src/test/resources/complex-types-table";
+
+  /**
+   * The list of dimensions in the Delta table {@link #DELTA_TABLE_PATH}.
+   */
+  public static final List<String> DIMENSIONS = ImmutableList.of(
+      "id",
+      "array_info",
+      "struct_info",
+      "nested_struct_info",
+      "map_info"
+  );
+
+  /**
+   * The expected set of rows from the first checkpoint file {@code {@link #DELTA_TABLE_PATH}/_delta_log/00000000000000000000.json}
+   */
+  private static final List<Map<String, Object>> SPLIT_0_EXPECTED_ROWS = new ArrayList<>(
+      ImmutableList.of(
+          ImmutableMap.of(
+              "id", 0L,
+              "array_info", ImmutableList.of(0, 1, 2, 3),
+              "struct_info", ImmutableMap.of("id", 0L, "name", "0"),
+              "nested_struct_info", ImmutableMap.of("id", 0L, "name", "0", "nested", ImmutableMap.of("nested_int", 0, "nested_double", 1.0)),
+              "map_info", ImmutableMap.of("key1", 1.0f, "key2", 1.0f)
+          ),
+          ImmutableMap.of(
+              "id", 1L,
+              "array_info", ImmutableList.of(1, 2, 3, 4),
+              "struct_info", ImmutableMap.of("id", 1L, "name", "1"),
+              "nested_struct_info", ImmutableMap.of("id", 1L, "name", "1", "nested", ImmutableMap.of("nested_int", 1, "nested_double", 2.0)),
+              "map_info", ImmutableMap.of("key1", 2.0f, "key2", 2.0f)
+          ),
+          ImmutableMap.of(
+              "id", 2L,
+              "array_info", ImmutableList.of(2, 3, 4, 5),
+              "struct_info", ImmutableMap.of("id", 2L, "name", "2"),
+              "nested_struct_info", ImmutableMap.of("id", 2L, "name", "2", "nested", ImmutableMap.of("nested_int", 2, "nested_double", 3.0)),
+              "map_info", ImmutableMap.of("key1", 3.0f, "key2", 3.0f)
+          ),
+          ImmutableMap.of(
+              "id", 3L,
+              "array_info", ImmutableList.of(3, 4, 5, 6),
+              "struct_info", ImmutableMap.of("id", 3L, "name", "3"),
+              "nested_struct_info", ImmutableMap.of("id", 3L, "name", "3", "nested", ImmutableMap.of("nested_int", 3, "nested_double", 4.0)),
+              "map_info", ImmutableMap.of("key1", 4.0f, "key2", 4.0f)
+          ),
+          ImmutableMap.of(
+              "id", 4L,
+              "array_info", ImmutableList.of(4, 5, 6, 7),
+              "struct_info", ImmutableMap.of("id", 4L, "name", "4"),
+              "nested_struct_info", ImmutableMap.of("id", 4L, "name", "4", "nested", ImmutableMap.of("nested_int", 4, "nested_double", 5.0)),
+              "map_info", ImmutableMap.of("key1", 5.0f, "key2", 5.0f)
+          )
+      )
+  );
+
+  /**
+   * Mapping of checkpoint file identifier to the list of expected rows in that checkpoint.
+   */
+  public static final Map<Integer, List<Map<String, Object>>> SPLIT_TO_EXPECTED_ROWS = new HashMap<>(
+      ImmutableMap.of(
+          0, SPLIT_0_EXPECTED_ROWS
+      )
+  );
+
+  /**
+   * Complete set of expected rows across all checkpoint files for {@link #DELTA_TABLE_PATH}.
+   */
+  public static final List<Map<String, Object>> EXPECTED_ROWS = SPLIT_TO_EXPECTED_ROWS.values().stream()
+                                                                                      .flatMap(List::stream)
+                                                                                      .collect(Collectors.toList());
+
+  /**
+   * The Druid schema used for ingestion of {@link #DELTA_TABLE_PATH}.
+   */
+  public static final InputRowSchema FULL_SCHEMA = new InputRowSchema(
+      new TimestampSpec("na", "posix", DateTimes.of("2024-01-01")),
+      new DimensionsSpec(
+          ImmutableList.of(
+              new AutoTypeColumnSchema("id", null),
+              new AutoTypeColumnSchema("array_info", null),
+              new AutoTypeColumnSchema("struct_info", null),
+              new AutoTypeColumnSchema("nested_struct_info", null),
+              new AutoTypeColumnSchema("map_info", null)
+          )
+      ),
+      ColumnsFilter.all()
+  );
+}
diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java
index 4e1c2566f02..4c1b57c434a 100644
--- a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java
+++ b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java
@@ -54,7 +54,8 @@ public class DeltaInputRowTest
   {
     Object[][] data = new Object[][]{
         {NonPartitionedDeltaTable.DELTA_TABLE_PATH, NonPartitionedDeltaTable.FULL_SCHEMA, NonPartitionedDeltaTable.DIMENSIONS, NonPartitionedDeltaTable.EXPECTED_ROWS},
-        {PartitionedDeltaTable.DELTA_TABLE_PATH, PartitionedDeltaTable.FULL_SCHEMA, PartitionedDeltaTable.DIMENSIONS, PartitionedDeltaTable.EXPECTED_ROWS}
+        {PartitionedDeltaTable.DELTA_TABLE_PATH, PartitionedDeltaTable.FULL_SCHEMA, PartitionedDeltaTable.DIMENSIONS, PartitionedDeltaTable.EXPECTED_ROWS},
+        {ComplexTypesDeltaTable.DELTA_TABLE_PATH, ComplexTypesDeltaTable.FULL_SCHEMA, ComplexTypesDeltaTable.DIMENSIONS, ComplexTypesDeltaTable.EXPECTED_ROWS}
     };
     return Arrays.asList(data);
   }
@@ -116,7 +117,7 @@ public class DeltaInputRowTest
         }
       }
     }
-    Assert.assertEquals(NonPartitionedDeltaTable.EXPECTED_ROWS.size(), totalRecordCount);
+    Assert.assertEquals(expectedRows.size(), totalRecordCount);
   }
 
   @MethodSource("data")
diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java
index 3fe42676498..e6bcf9f5fc8 100644
--- a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java
+++ b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java
@@ -84,6 +84,11 @@ public class DeltaInputSourceTest
               PartitionedDeltaTable.DELTA_TABLE_PATH,
               PartitionedDeltaTable.FULL_SCHEMA,
               PartitionedDeltaTable.EXPECTED_ROWS
+          },
+          {
+              ComplexTypesDeltaTable.DELTA_TABLE_PATH,
+              ComplexTypesDeltaTable.FULL_SCHEMA,
+              ComplexTypesDeltaTable.EXPECTED_ROWS
           }
       };
     }
diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md b/extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md
index f1ac54fb8b3..f45b33ab62c 100644
--- a/extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md
+++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md
@@ -84,3 +84,14 @@ python3 create_delta_table.py --save_path=employee-delta-table-partitioned-name
 
 The resulting Delta table is checked in to the repo. The expectated rows to be used in tests are updated in
 `PartitionedDeltaTable.java` accordingly.
+
+### Complex types table `complex-types-table`:
+
+The test data in `resources/complex-types-table` contains 5 Delta records generated with 1 snapshot.
+The table was generated by running the following commands:
+```shell
+python3 create_delta_table.py --save_path=complex-types-table --num_records=5 --gen_complex_types=True
+```
+
+The resulting Delta table is checked in to the repo. The expectated rows to be used in tests are updated in
+`ComplexTypesDeltaTable.java` accordingly.
diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..a56f68f447baee7e6115aad6e4ccb68586175fde
GIT binary patch
literal 20
bcmYc;N@ieSU}8vExl-+maj;$ag?ZBfJy!=u

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..6b7e86bcf54a017d8701a0f47f79e34e0d195fa8
GIT binary patch
literal 36
scmYc;N@ieSU}AWpcyWrDl6iFUzYoe4`yQ$3YabQ){QJ_}xX|U#0P?jD@c;k-

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..88b089e950346e420d746c1643b8066e824805ec
GIT binary patch
literal 36
scmYc;N@ieSU}A9Wb6vu1GAH@3#S$0$)Uyw=TO66wjXf^h3)8y@0NEi8w*UYD

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..7f4972520056438a8fcb3fe998edfe7d90c9c2ba
GIT binary patch
literal 36
scmYc;N@ieSU}9Ju*|hMoT={`_+Ix=)+>sN})jm_CH|1-cb6)o~0QV*jApigX

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..cd8fc7a087f660653d9af8d14911dbbe31a7a752
GIT binary patch
literal 36
scmYc;N@ieSU}8|*n37}aTBY}wL1<<+%e~pmEFZ4vvu|kVyLzV@0Ll3cd;kCd

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..038082a933b714e530c2c4cea6b49f5b391f4589
GIT binary patch
literal 36
scmYc;N@ieSU}A{lNPTX?c*${h!bFyh?NT>Z1^zgjZtQX4UYOoR0N*?gv;Y7A

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc
new file mode 100644
index 0000000000000000000000000000000000000000..311d2a22b04d925d5e2f9cfdb60e405bbaafd525
GIT binary patch
literal 52
zcmV-40L%Yla$^7h00ICC87fl@eX@`q3>oWVEG+?7lFEsb7v-zj%vt*E#AO1vFwe6v
KP0fF9Q|IN#5*5_|

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json
new file mode 100644
index 00000000000..84803f9483c
--- /dev/null
+++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json
@@ -0,0 +1,8 @@
+{"commitInfo":{"timestamp":1723511561738,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"17937"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"b9eae5f4-d55b-4c38-b365-8228ec09248e"}}
+{"metaData":{"id":"ce998219-9bde-4831-b78c-14b11f919fbe","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_info\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"nested_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_info\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"float\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1723511559184}}
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"add":{"path":"part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet","partitionValues":{},"size":3288,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"maxValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}}
+{"add":{"path":"part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"maxValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}}
+{"add":{"path":"part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet","partitionValues":{},"size":3289,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"maxValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}}
+{"add":{"path":"part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet","partitionValues":{},"size":3290,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"maxValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}}
+{"add":{"path":"part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"maxValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}}
diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..7a105c0f74fff6b494898d601cdbf192592f6b05
GIT binary patch
literal 1488
zcmd5+&1%~~5MET<NnQ66*s_Km%z~i-S4h}y9Ft4wr4&;_>8%8_){fmJE6I^oDaQDX
z`WShSzDN%p{a8vd*v_F)&_(m}eKX(8?DhLEF16af&px)|rHF2)vG*N(L7Rlux)!sP
zBtwfeMHGqER0Z>pTC_2_B7_WUtfp0Ls70uSp(mihhrp!qFJcoVp2-tx6%=h@U@Z_n
zbjjp(3SzOd0Ad4j`Z%SrKc$hUP()6dn+1YGa@s`0%fcYTWTqh{IrYVITfQ~v<>f!W
zh1K+4=mb)vR+q46XelbArwCNfyyO*Wx7t0%!f0WOr9g%3Of35zqF99itM~1p?GSdp
z6A|BO8+E-)z}Qjv!}h@lDU4vhPS)J4mcYk6@0|DeT!EG`AAei(R3mgLiRL0qhrt5>
zf<$Y9ZSgT`3_ZR@<m2^b(<?KpiIoT?Vdg_~F|n_C87LntAVRwr21ckL{)|*P&E{y{
zM`PS2{wJMgQk5zpg*vR$&G8NGuIGQ;S2h2<>?f64ol%*_tED({@BTWy3MW5B{ND?6
za4ZJ{Dgm|^U)E5rKYGxJr(P_6xiZ<c7sj~`g~71Q_Tk9~akC-pQ*roj{isvC6dl)g
z`?lM8GfR|~9XT7gqrU50%HiyyKLBrd*>~jV!tta4_niv^XC&oJoVS1CYq{qm4|PJW
H@h$%YG+oZJ

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..bb2fb67389d2373731fc431f3a626e6744c61f53
GIT binary patch
literal 3288
zcmd5<PiP}m7=M{erjs<g)n(sfMxvB~hAhM(vu$>>Ay~vkN-3p?^dd`{W?tK{lcbwW
zR!b=qU5ZFm^rCnw*ke!X_8?f)lLxQ%uotC=J;vfe7L;AT_hvHlN<-82515Ae^S$r=
ze&73j-}_!V_ws8NMmU2<@i+IMymKLgC3sb^iqORg2_Y1Z!G(XoUWUK#Kiv6E$zYYc
zDF{8Uakq={m?A$xu~?o>;3l|((+k+ZZ-4yo83B2u6o;`mbMimnra=sU{PNMgVZkq*
z5BwNp;a|og_&NO3C+|O&1;XJRCuEl}_Q%_wf0q<|g5Z50gpMa6t}?FR<1gL?^<;Qc
za3qTavfNA|8Srk?H7mGZ1wR$1@x5Q~@1`ptIoRbiLYoYX3AS^zxz503A8e$=1(kSo
zBOsmrtfXs@BmW~jVo4_m@nOb=bWLWH1oMDdv4XC}3je^rN6Xkx!yHeB!Gdx^uuL5Y
z7H(HXGt&gjTpHgyLsUEqYeag6%;S~d=^z#ww<SCsSP!Mpx*HFz-8h~K9QI{A7d&ny
zLa-ghp9X>ad1)%$UbS1d7uwYoTDNfuVR=l_q?&_@l9sYtEqiUD)>vvn7L}O_#8s3v
z*{!ubEW!H0b%?sOP8%NcA8vcCPSx`#$PpAygQMKA>vRmwWEhB~U24#_N1X+6igNZ8
z#h(J@#F<k8LJi1q3K%W~2B+CsbSWcI>-K6dd}$`{ho6KjEZnu-4rNH`F{H)85=4eo
z07~#tL?k_<?`2geRlTSpMQ6HTiAZFc%x2X}RHD%nLAkI5BP(PRN*}^Th#aNxWztpf
zODH=J8&4kt;F6#;SGT7su&HE)tYqgaeyOu!0T7hqh3~(K=)W`cy&-I(>_IFHJSS#G
zT*N`_Z?59KVNA$_A-0NbDTC9#Kqq*3!ns_pl4<a&z|Mm&#Ln+x)H@9AQxzL1dkAQO
z+_^qAA_jj5kbT4+oPvEu_qED`-;GDX_n|wvhTZrXYxt3YKH_WW0;uFGQtAU5)Ui;%
zT&0_2D%%AHp5Z9ZEZ?W|e8kv3OR_xgLyu1lWIMSzIN=PLgUO+W$CEuZk0&DyMXQ*y
zM=XFJl1#&`1k;e$ak$~4O~y7C(8**8%6KbLxdye%`XKLymI(Pl8X&h)Y)w1t4A#eF
z*cyEx{r$;7-y)ka@XmG<879)^q<@9@g*T+<`Dp_!3pxzng}aRFE5=2>Rfv>T$wTEb
zbltqBHCxN3y=p^4G<k0{@6uMg)@*2rakFR^kap#iNBfRzLL0r)p`O`lzG>E}XFIlM
zYd;UI8{XO~HOhwH=#0FvRHLrbHp*|T8>}V5P|X2z;N~_P%kb9dxULOd%qT-krFo-=
zXq4ACHuB-js6mGaAt4_FbI_n)_rrj0NtbC0?A2z&vulm^EKvD08|UT~y7p<}|74|H
zs!%1Ms14TX*7+B7`ZW3<uPe&GANEs)8l9TqG$SgCbJpF7lQzQfPhNOG6J}sq20BCn
z^bYtihOqo$1dPyWg2E3=h9B;Mamj7k9=LJaD^I;4ZfziKI@tIx)?1@Nrx>+NYuvO(
zbBmpt>x?>!6IN;5D%^02i`T~|Xtj7_yx^3s7pjg;t!oyYD3ly$(Y~VZ!H=UezUa#&
JLUZsx&0n-Y%g_J-

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..641396cb6f7cf670208d1da2d475f6dcc17d490a
GIT binary patch
literal 3291
zcmd5<O=ufO6dsM%>y^BTX~+z_MJ;B*!U1*hR#u`IQCu2|F~*oud?~@Ky=z;^N^+!C
zT4M~k6iV>Hg`P`Vs3|6=6j~@gxU|srQc5ocUlMZiq0k=GTX6DbX0<!U#<EKbU4(Y#
z&71FgfAdC~ee)fMAd(|P<nX6ozaGdD3{#b82wfb*2%&T;1>fue`8cVN{rgWlnH<r$
znu^dX23Nb7PN_GZBc{yf@K(g8SBXWw`}5~7R0(^qoQB0T=K3&V;09Rn`{!SF`XoKQ
zAn7qf|5Y$bUM5fe{`kHwF%FDy#x%DqBK+*fgF{&-vYsjOX9DzCX#RTi(S;mQV5X8J
zGMGgs%cF7IUvpSg;Qc!2X+$UAeE;ErUV*5CT+v~TfiW*1JiWKhz!U*CSmt5H+`2)C
zo%s?r4Xv9TX-Ke15#q}VTQ&`aXCe{z*p+fjBjrdi=hg4hGIkjTrtFyjW5o!ZO{`K_
zbii0197!y8Fc-SoIjWHv@DZ6gI!A6rqrDWFjfPv8Ohv<e97}i7v9yyWlaa!%5-V)>
zk(EfgtqKOP@7bkEy|wB#Z_l?DSNxhwvWO_d*uYf}m9UX@n@x9ZzFJ>uKo-^M3fNUr
z3?-<xLW046Q0?~vzvkCNra#aMo9)F=Y)}#?qJyGRcWeGIn$9s0Y+S1Qt<d-8<t^&z
zTU4<H%84^)BZMlDlPoY?hy-4vy%6|}M60>0UGwpDL71O`EX?0^gSO9*^kHPAK@wR;
zGyv-3tB7JVXFkkpP^xB0L#oN*LNLX2iq7P<N>ZZ9h@z3Wq!KA)6G|V<Mu;9{$z|G6
z$r~s?2Zz%f2H>(3X&&8fRDn&SEA&==t|Cg^-~|XQyzs?LMt{c8AN3O(<@Zvt;UjWp
zx=u{fS<v~1UTh5zN}u#IFSc-w=psNzczD8jxZb8ypjCmB2OqGVzh$ep8QP~Bu~2>=
z(4ugUoKPdO@%sRILa=+MV3*PDY0UVYbYlDobVq&INuT1wZw&O1`_KVUEFw}q0W_*(
zseXBsZqUhm2N-yUlRUFdb3D%njO{^&<#|`hk!(-4qlbeNvI5vuAcnxeo{40{Vkm^N
zR8H9;GvJpDi{U0*Yr3f8_`oHHj2$kZQ|J<10oBS?s9n|vc{g-q$WL^D+{`kcw&6zA
z%wYwN#tD$(d~(na=)DwZ=izEKcpnwtV81j&W`W-}(6XSz@Kd<MxV~jv^aqvVyhi^}
zFGJTY7)GPHY`d#2G(?;CM*FVcY*ia|qpxU}>``P~Ii}IN6WGv3@3j5UZZ_VxYkuf@
zZs;11``4{-ZPmBNEz#(#g0)oj1FvO`zqf9&mIzB#55$pL(5NrNtlka+7rK}=4lUI$
zSY1SGe0^i15bsPHbc_%a@-+}g4f<VS26Rh**>8e;u~84*YP~fBRDx#X%)C<9K2H3f
zc*;W+M+qpZz2kKA><%5jjsC}eCHWU-pUc$b){M7VNl~1!?vC8F368jVVSXx1&$RSZ
zNCMEk;A0!&@`n`=LZ^ueKQ0-5xktjKpy7s~#$|7v`#{{>K*m&b@L#JnhoVj~<k(Kp
zc7{e4+SR}t@)pLNa?u&R?v)m<703L=()Hq~SH3p7=()ag)$zwh%bvI3UNIlSkE2sQ
M=&KAuv+%d(1tDO@#{d8T

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..abee0ea5d0a95f6a5282777b5b7c27e97f96ffa4
GIT binary patch
literal 3289
zcmd5<Pe>zI7=M{e#xX{v%f81<Ns9~^vXBl&&2Cmgv2-aSB2tQ%vdHL6G;IF3$z)p*
z!J`MEp1Un{JxHNTDO+egcq)a$9z5DZ7Ybz$d(5rNQfPng&17b*Vzji-L6bM%``+*O
zz2En}@9Dxj@8>C@X*xoG`ts*Ti8Ph)s!)}Xp-G7l5|8mu{=hy(i}b7CJKqncsVdwQ
zLf+7X+fY2F$bXW=U>2@2eiPo3H>p8iescUojUW#f<Cu&~Bm%7=hG!2yf7l=KOXYmO
z7~ID}_^*O7`Z_&&dh}>0!Z?r>jByc}Pk6ld%Wg8_<44aO`J+w?VM?^~WCyy)f73;R
zE_aebk$cWYo-2dzarmh;MSs}-W^b?r&0&{QgiLcVzRNFu+ur10GI9Zthl@gFNklgc
z?ey0qT~oXIMTS(;86jd?=F_?+i%f*(9;;&Gx)vLcz+Bb8OUv1#FnGnIP9xavFfuK3
zbr7r&Ckz(4Ok;)k^1ahcrSsS$(bH^^mi^IDj4t@YU5U>6!xJg69>fD{Cr)R4heJ7V
z*h$be-@31O3}F1@+DxjoZZ+>MwN}=gnnja@%A=AdRcum_w4~K+S{qB1dbxotDsv^o
zRgg8gT4}jd!v5jf=Tx1VQ+K)lK+A2mS6pv_9753)9Ob%Ib4JNrnuAE%wYt-C9eXLd
zM7em0;w?cr3Ff4aPysnjf??1%*p2pb)!`&+&06n<FU{q=@cWU4r3Y5E?Qo>jDAD4u
z_>oZ+KnXERn53um#~B=MSudzW(YY>EViKEW^BJ`imS{L)BpZ}eXhk+r`Usm4_B=_i
zvyMXFCYePXJbe_v#fZ{E-L9%&Q`s6TXBJCdscWJD!6FLZdyAsK<mg}bQIlkjVnN{9
zXlA;bm}3j@`G>pM9iWUo>*KrFmC`ij33Q4_BwWb#9-D<%2`3Lfh+TX~qu%FepQ_X#
znG>M-a%az}k!bKIfILU+-YGcbbVn)={vaL(e-7R0JsiX@*uyageI@qL0jT6DQalIL
zuj5Gl3YG4#nM?-^BEw;xd8av>=jWX5vwoiELpe>8J=snl4nfEZ;82E!V8EkJha;}x
zS!*<9uXq4Y`?-cYxL8wO9R~+4JY@WEflg*+wg%VIP1G*$gQ6SeqmX+kK<*^@p7s@7
z!uqI;qj3(TcRmH^$80+W?@T9=_WG#ziTFiV_1=h{6Soao7CMYLg$JDLJI=*^RG5@e
z*$d@5x^7O>8qHPHTDQ;;P0<_82Trq9Y1FmEgjq1hh<4+QN9%soL>s-|c3iXB_{gj|
zu4P-UrTyBsX}BBfjxl9;jn2p!<%(0aTgKFfn+9)*n5x(?`)<`neHCx@cC~7uiy2dB
zsZP%5A{tYhTU)tcW!Rttgn&@Y!R$Ba54<qYEuB@T3HwT;?pl?4YaUb{&DN!PBVGF}
z@qe;YAyuFfC@Q^mx_kKzoxP3z$NLKN?}dG(P{Ug@SZ0Jpaml(nb<>78-pz~m3t@Vu
zrKdwEK=*<VVhG9~Gccml_=O*ojJVu=W4YR}T(}9_yH`FCcejW(>mU5LYRwV9Q;g)z
z{Dhew$u74mReQu<p3D~~@?*E{!t$+&NoS>Sdt%Hk-Wpr6Ehm37?@W#rZF||ep+CkS
QM;H8%Z~F;Zz&|yA0a8}Q<NyEw

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..453c3879fa35afee8fd369d640be632857876386
GIT binary patch
literal 3290
zcmd5<PiP}m7=M{eCet+A)@9#gMxvB~hAza=Y?|F{2vTv8Qc5Wzy~t7~ooO3(l63QD
zwUk0ZLG7Uj@v<PYN`)TwAbODsJ$dx72n+S%qK7@`WywJnDd_LLnaoUSX<AV*O_O=w
z``+*Oz2En}@3r%<y_utgrsybr|MBxrqA4n2s8E%Vp-G7l5{*Rh$zQOS@N;<ok1sE!
zs4CnPLSE8@+fX#3$bXSYJR`$5hF{F_H7`?xe%JlwgP{=gaFK&XB@)8c5X7^ec7KnD
z{E`{ZF9P@czYWIdi}a(t2alBy<KVbpjEcxS!uNKb{}>DTc+vM_ADMz*iI(ZJuRnw^
zGDZpwgpnZ3tpwq!+_Bel-uE^5sWeGH|Kk3m!7@aLT}~3R#lg5_4}RU+;9#-~He3`c
zOCq{qXs5p*>6+S$A@oQkoe?6YWj?KIvdBbW?z1YA*R@DK1andU9xZ1N!(fQl$P|L@
zJ%*-bt`365*R`;jYXY;7#@$1TsdNr|BzlT1&{gm0AVTN8#~q2zc#nsYZ{3Uf)^3ze
zdk*`u@30;7!FCmw0f8J{o=&#b%;ufN*3zn7GiicQc}&uzsznNtmN1)5bA7Q|Uv40a
z%4`{N6=Y3zs;xGauz$G5ZO5+J^)~k(Y_*%6rMA034xnffj&j|s*<)lj#X%(Pa@}sV
zZEG>SL^*qj;x0is3Fd@{Pz5<nfZ>v7uo|6;V{;O<X0G+Zmu53=_yfqo;yu&p*c>T2
zMzkm_USw1SP)y7cCg~~tVOmA0>IIc3I@g6tOky)^F0Gb>5)Gah8TU&nup*l%eS}R2
zdzzqESXZI1lJo)&o<0WPVn}JBZckOPsce<4rWeX?scWJD!6FLZ9fi^V<mg|;X_ll9
zB7WfG;mq_lG0W!R^AC5iGe{ZxInH;nBc*8473dg`NVt&e4K@R>GEN@eh+TY#qu%9c
zpQ_X#=|iA-a*v-<BjMl=0eOnp{Zp{d=?+vL{2p$FUht>T9pA%V^o%_`;-E)j4_$uM
zxQY}{0rl!QRKG%{TWmVr1%t?NkZ0a$PUiV3XZvh`=XqaF(L`Ui<A*~K@&eeGp&=OX
z8kq{7xP~XK;glWm0Dc(Y8gAnPNxF6HAGqL<@xui=nJuG?w`1k&s9oL%MK{caAwNz6
zay!BIw2OO7)yHHUjZ+}q^C>{zWm^$=r@OI~+eh6`#4oz4J0f~U+%{-g=rH0G?s2Yf
zIT!n0VNzOUzbRMHbu*gQXs%?<H4_anD|%!0p51I!8+9#~&la*1M7w&zqjlHGqK)3|
z*zIhy@lLj8w@u4zo7$83hS6SMvyGDBHaa6?ELUyEY8j=sHw@kqF;%r-_S~FCeFdX>
z$8k(_F{6Z*YG;fdqEXt|+|2kZg9hy*_=I8(X0JiN<%WT7X|LE#*q0jhwpp#W=0N4r
zY@VAp)U{6%|0g>YQu!)@qS{}l+vnfV$=m3EyssetZrB$JHMlkXWkygG=d8P9H*J99
z-n<x}3DY+%eH{V;dKbJOgJ1rbff1d?D}29X#O3Z8mz{>$hMTaxe&GXgdy{B0-obyR
z)*SUZ#b_>@%V%?=<CRX;u|}=RWUiRcP298!l^gj<d#P|UKVcPbOe|TZox7g1Cnt)Q
WRWYyX5Any*8E^8<03q}EujX$sv&okL

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..aadf1e152c8713e73e04bcb969e5d14bb6098a6e
GIT binary patch
literal 3291
zcmd5<K}Z}|7=AlDJC5rn(%^f{lx>j#7X#bDt-HxamSSlrA|g_Xml9-jM%`p~*X+)w
z6%lgip`{)?<dj43<iX~Y(o%{A52YoSw)D_L4o&Hy<j_M}gj^))_ukCTj3};_7CPwe
zoB#dq`~LU+|NEcLy!lRn5}Kuh^ou`#{8GwN39kxO2{~Vs2qDRYj8Fc+K1M6_;FoW|
zPG+ep+!R7y(S+OiWJ0<6f^g>)elbCM{6())gMR$;!%zET*!|@sCX<rz$B2V#h~nYX
z7k~A|{4)8#F9Dw-oc>iXMqj2Me*XKjbd0fYSTH6<WC7tPckkWljrj!8b4UIRjKhyh
z^Yp<-=dx7Bn?fUHB(6+a;N5Um3tS<5zYaf@X6W4?zI&!rpgQbwhL9}|#&@~(+wC<D
zCi`IhWg)X9qU(ot`b(0osh#A+hE&oSA!1tQ)4C>$Oho1`s}e<BOB7=;XZ7#Ua`re3
zUhxc!BiPO`HZ5~?5Uda<3Kk$+SRuXw?<`a4H1<gJESsg*gVA1s&IH3viB1N?M^b3r
zNru+_B%KHxcID7vBSn`3>$c)EfbrnMM5eW3HE+zd7M7iwMbm`JLy{&{ZBml7wAE}{
zt8>-*Vgp%JrYeZ5Bx|x;ZFy9}{^8o=xK7Qfd)&XT<u%(2p1(kjplAk;a^0#qLu4w;
zK_u-$-D!D_Jr`f1oV-Nwm!O;kb2>n%f}Ezoa4s;|jrP3ja1ym<t#rbdrt*IHy~x7c
zEz4~?94RwIv?MG+WK;!EO3V@_=~?}LPDQEeC6y>T*M&+<Vv}q-r&gj8jYf<Nhb0wR
zkxi67!X|`0Nz(zguh2J0ZWaem9|CYWrnFGEqbk@`w#=^QW-ETFYoY+bA`0Jsi=!WK
z^anlEB)Ppr82E5JGo4LLu^IUM!(D9lQO17l;k(#GXU+Hm9pVuQ7jnJLCgD}V$-@s~
z7vJ%yw>jE9l^P`X2xx)a!^hM}Jora|JVxy9DcI$7dnylp2mQVi{4sQg_pp;ZVGloX
z(C1<g`v8@EMasv326Y^(U!l@fHj&#0gUE1{XWnU!=J^R{+v??c-j%a7-IeX|;Shwp
z0Cr_)2nIX@<I#v~c+?tC*>fJi_q|-hjTF~V)Ny#=qC>_H7wBZR$d=(+xrEx~eNc46
zLLBl=29O(RzNc*k*RVb$<7gZM>7P#l`T@J0fOl>`mG%3m|B3iTSM}eBo)@<bS{6Et
zIE6c$>nqO1zEPNzQ`sNN0J?5o(;Ces(^|365KYk=&09{hRc+L@)Tmi9M~HUuh)3(D
zYod+bY&)LWY`kyQ9M7^X&(a?EtQp?wiero!exoz;#$wfR?Uph2-kQN%BBrV~%z>NR
zs4wBI-gaFJUCbCmOLg)_2hkW?TVKzIE29P-B7}ru4(6aizw3vAZs{yJP1qM2b<e8S
zThpNOY1U888|&IfiT{(G3aLVsKvC_k)3wuY=;&?qKi*fAe?ROqg&N(O;W8sCic{9z
zp_?|s@o!$dp9s@6EnOWV0Xi3a7(-b8n1K<UCMf){WW?nj7#H1!<-twZ-a7MvxVBET
z$>89>R%;Fhono+H7Dmm&;P8CA>e_?$e6dg-EsR{TOY@gUi_Svn%IJt)zC5yETTbCp
c!6}ZEZF}CjsNcsQM<@J{&w2@&!M`<s1F4tHTmS$7

literal 0
HcmV?d00001

diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py b/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py
index 34a64977304..a116513b01d 100755
--- a/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py
+++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py
@@ -15,12 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-
 import argparse
 from delta import *
 import pyspark
-from pyspark.sql.types import StructType, StructField, ShortType, StringType, TimestampType, LongType, IntegerType, DoubleType, FloatType, DateType, BooleanType
+from pyspark.sql.types import MapType, StructType, StructField, ShortType, StringType, TimestampType, LongType, IntegerType, DoubleType, FloatType, DateType, BooleanType, ArrayType
 from datetime import datetime, timedelta
 import random
 
@@ -39,6 +37,55 @@ def config_spark_with_delta_lake():
     return spark
 
 
+def create_dataset_with_complex_types(num_records):
+    """
+    Create a mock dataset with records containing complex types like arrays, structs and maps.
+
+    Parameters:
+    - num_records (int): Number of records to generate.
+
+    Returns:
+    - Tuple: A tuple containing a list of records and the corresponding schema.
+      - List of Records: Each record is a tuple representing a row of data.
+      - StructType: The schema defining the structure of the records.
+
+    Example:
+    ```python
+    data, schema = create_dataset_with_complex_types(10)
+    ```
+    """
+    schema = StructType([
+        StructField("id", LongType(), False),
+        StructField("array_info", ArrayType(IntegerType(), True), True),
+        StructField("struct_info", StructType([
+            StructField("id", LongType(), False),
+            StructField("name", StringType(), True)
+        ])),
+        StructField("nested_struct_info", StructType([
+            StructField("id", LongType(), False),
+            StructField("name", StringType(), True),
+            StructField("nested", StructType([
+                StructField("nested_int", IntegerType(), False),
+                StructField("nested_double", DoubleType(), True),
+            ]))
+        ])),
+        StructField("map_info", MapType(StringType(), FloatType()))
+    ])
+
+    data = []
+
+    for idx in range(num_records):
+        record = (
+            idx,
+            (idx, idx + 1, idx + 2, idx + 3),
+            (idx, f"{idx}"),
+            (idx, f"{idx}", (idx, idx + 1.0)),
+            {"key1": idx + 1.0, "key2": idx + 1.0}
+        )
+        data.append(record)
+    return data, schema
+
+
 def create_dataset(num_records):
     """
     Generate a mock employee dataset with different datatypes for testing purposes.
@@ -94,6 +141,9 @@ def main():
     parser = argparse.ArgumentParser(description="Script to write a Delta Lake table.",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
+    parser.add_argument("--gen_complex_types", type=bool, default=False, help="Generate a Delta table with records"
+                                                                              " containing complex types like structs,"
+                                                                              " maps and arrays.")
     parser.add_argument('--save_path', default=None, required=True, help="Save path for Delta table")
     parser.add_argument('--save_mode', choices=('append', 'overwrite'), default="append",
                         help="Specify write mode (append/overwrite)")
@@ -103,6 +153,7 @@ def main():
 
     args = parser.parse_args()
 
+    is_gen_complex_types = args.gen_complex_types
     save_mode = args.save_mode
     save_path = args.save_path
     num_records = args.num_records
@@ -110,7 +161,11 @@ def main():
 
     spark = config_spark_with_delta_lake()
 
-    data, schema = create_dataset(num_records=num_records)
+    if is_gen_complex_types:
+        data, schema = create_dataset_with_complex_types(num_records=num_records)
+    else:
+        data, schema = create_dataset(num_records=num_records)
+
     df = spark.createDataFrame(data, schema=schema)
     if not partitioned_by:
         df.write.format("delta").mode(save_mode).save(save_path)

From 204533cade64bad591b31918ce1ad2b7de03b2c2 Mon Sep 17 00:00:00 2001
From: Laksh Singla <lakshsingla@gmail.com>
Date: Wed, 14 Aug 2024 10:22:19 +0530
Subject: [PATCH 11/99] Remove Query ID verification check from MSQ workers
 (#16886)

Upgrade/Downgrade between any version till or before Druid 30 where the newer version runs a worker task, while the older version runs a controller task can fail. The patch removes that verification check till its safe to add it back.
---
 .../org/apache/druid/msq/exec/WorkerImpl.java   | 17 ++++-------------
 .../org/apache/druid/msq/kernel/StageId.java    |  3 +++
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
index 7d2964eb2f8..912826c5c5a 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
@@ -1100,7 +1100,7 @@ public class WorkerImpl implements Worker
      */
     public void addKernel(final WorkerStageKernel kernel)
     {
-      final StageId stageId = verifyQueryId(kernel.getWorkOrder().getStageDefinition().getId());
+      final StageId stageId = kernel.getWorkOrder().getStageDefinition().getId();
 
       if (holderMap.putIfAbsent(stageId.getStageNumber(), new KernelHolder(kernel)) != null) {
         // Already added. Do nothing.
@@ -1116,7 +1116,7 @@ public class WorkerImpl implements Worker
      */
     public void finishProcessing(final StageId stageId)
     {
-      final KernelHolder kernel = holderMap.get(verifyQueryId(stageId).getStageNumber());
+      final KernelHolder kernel = holderMap.get(stageId.getStageNumber());
 
       if (kernel != null) {
         try {
@@ -1137,7 +1137,7 @@ public class WorkerImpl implements Worker
      */
     public void removeKernel(final StageId stageId)
     {
-      final KernelHolder removed = holderMap.remove(verifyQueryId(stageId).getStageNumber());
+      final KernelHolder removed = holderMap.remove(stageId.getStageNumber());
 
       if (removed == null) {
         throw new ISE("No kernel for stage[%s]", stageId);
@@ -1191,7 +1191,7 @@ public class WorkerImpl implements Worker
     @Nullable
     public WorkerStageKernel getKernelFor(final StageId stageId)
     {
-      final KernelHolder holder = holderMap.get(verifyQueryId(stageId).getStageNumber());
+      final KernelHolder holder = holderMap.get(stageId.getStageNumber());
       if (holder != null) {
         return holder.kernel;
       } else {
@@ -1240,15 +1240,6 @@ public class WorkerImpl implements Worker
     {
       this.done = true;
     }
-
-    private StageId verifyQueryId(final StageId stageId)
-    {
-      if (!stageId.getQueryId().equals(workerContext.queryId())) {
-        throw new ISE("Unexpected queryId[%s], expected queryId[%s]", stageId.getQueryId(), workerContext.queryId());
-      }
-
-      return stageId;
-    }
   }
 
   /**
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/StageId.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/StageId.java
index 5b98eed0da9..a928e5834fc 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/StageId.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/StageId.java
@@ -31,6 +31,9 @@ import java.util.Objects;
 
 /**
  * Globally unique stage identifier: query ID plus stage number.
+ *
+ * Note: Versions till Druid 30 had a bug in the QueryKits which populated the {@link #queryId} field with random
+ * UUIDs. Therefore, all usage of the field must be vetted instead of assuming that it will be the expected query id
  */
 public class StageId implements Comparable<StageId>
 {

From c2ddff399d7b46ea7c220536a46fcca249e8d0df Mon Sep 17 00:00:00 2001
From: Maytas Monsereenusorn <maytasm@apache.org>
Date: Thu, 15 Aug 2024 02:31:38 +0700
Subject: [PATCH 12/99] Fix Parquet Reader when ingestion need to read columns
 in filter (#16874)

---
 .../druid/segment/indexing/ReaderUtils.java   |  9 +++++-
 .../segment/indexing/ReaderUtilsTest.java     | 30 +++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/apache/druid/segment/indexing/ReaderUtils.java b/server/src/main/java/org/apache/druid/segment/indexing/ReaderUtils.java
index 5298f2605d8..cd49757ee10 100644
--- a/server/src/main/java/org/apache/druid/segment/indexing/ReaderUtils.java
+++ b/server/src/main/java/org/apache/druid/segment/indexing/ReaderUtils.java
@@ -27,6 +27,7 @@ import org.apache.druid.java.util.common.parsers.JSONPathFieldSpec;
 import org.apache.druid.java.util.common.parsers.JSONPathFieldType;
 import org.apache.druid.java.util.common.parsers.JSONPathSpec;
 import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.segment.transform.Transform;
 import org.apache.druid.segment.transform.TransformSpec;
 
@@ -130,12 +131,18 @@ public class ReaderUtils
       }
     }
 
-    // Determine any fields we need to read from input file that is used in the transformSpec
+    // Determine any fields we need to read from input file that is used in the transform of the transformSpec
     List<Transform> transforms = transformSpec.getTransforms();
     for (Transform transform : transforms) {
       fieldsRequired.addAll(transform.getRequiredColumns());
     }
 
+    // Determine any fields we need to read from input file that is used in the filter of the transformSpec
+    DimFilter filter = transformSpec.getFilter();
+    if (filter != null) {
+      fieldsRequired.addAll(filter.getRequiredColumns());
+    }
+
     // Determine any fields we need to read from input file that is used in the dimensionsSpec
     List<DimensionSchema> dimensionSchema = dimensionsSpec.getDimensions();
     for (DimensionSchema dim : dimensionSchema) {
diff --git a/server/src/test/java/org/apache/druid/segment/indexing/ReaderUtilsTest.java b/server/src/test/java/org/apache/druid/segment/indexing/ReaderUtilsTest.java
index 6fddd3b4d65..b5bdc345889 100644
--- a/server/src/test/java/org/apache/druid/segment/indexing/ReaderUtilsTest.java
+++ b/server/src/test/java/org/apache/druid/segment/indexing/ReaderUtilsTest.java
@@ -34,6 +34,8 @@ import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.FloatMinAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.query.expression.TestExprMacroTable;
+import org.apache.druid.query.filter.AndDimFilter;
+import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.segment.transform.ExpressionTransform;
 import org.apache.druid.segment.transform.TransformSpec;
 import org.apache.druid.testing.InitializedNullHandlingTest;
@@ -364,4 +366,32 @@ public class ReaderUtilsTest extends InitializedNullHandlingTest
     Set<String> actual = ReaderUtils.getColumnsRequiredForIngestion(fullInputSchema, timestampSpec, dimensionsSpec, TransformSpec.NONE, new AggregatorFactory[]{}, flattenSpec);
     Assert.assertEquals(ImmutableSet.of("A", "C"), actual);
   }
+
+  @Test
+  public void testGetColumnsRequiredForIngestionWithFilterInTransformSpec()
+  {
+    TimestampSpec timestampSpec = new TimestampSpec("A", "iso", null);
+    DimensionsSpec dimensionsSpec = new DimensionsSpec(
+        Arrays.asList(
+            new StringDimensionSchema("B"),
+            new StringDimensionSchema("C"),
+            new LongDimensionSchema("D"),
+            new FloatDimensionSchema("E"),
+            new LongDimensionSchema("F")
+        )
+    );
+
+    TransformSpec transformSpec = new TransformSpec(
+        new AndDimFilter(
+            ImmutableList.of(
+                new SelectorDimFilter("G", "foo", null),
+                new SelectorDimFilter("H", "foobar", null)
+            )
+        ),
+        ImmutableList.of()
+    );
+
+    Set<String> actual = ReaderUtils.getColumnsRequiredForIngestion(fullInputSchema, timestampSpec, dimensionsSpec, transformSpec, new AggregatorFactory[]{}, null);
+    Assert.assertEquals(ImmutableSet.of("A", "B", "C", "D", "E", "F", "G", "H"), actual);
+  }
 }

From ca82ecd3528a68d4d31e9d6228070c4494ec79fc Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Wed, 14 Aug 2024 13:42:26 -0700
Subject: [PATCH 13/99] bump axios to 1.7.4 (#16898)

---
 licenses.yaml                 |  2 +-
 web-console/package-lock.json | 18 +++++++++---------
 web-console/package.json      |  2 +-
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/licenses.yaml b/licenses.yaml
index 0646c7131fd..b6e07572341 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -5301,7 +5301,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Matt Zabriskie
-version: 1.6.7
+version: 1.7.4
 license_file_path: licenses/bin/axios.MIT
 
 ---
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index e9319969b69..20396ed7265 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -19,7 +19,7 @@
         "@druid-toolkit/visuals-react": "^0.3.3",
         "@fontsource/open-sans": "^5.0.28",
         "ace-builds": "~1.4.14",
-        "axios": "^1.6.7",
+        "axios": "^1.7.4",
         "chronoshift": "^0.10.0",
         "classnames": "^2.2.6",
         "copy-to-clipboard": "^3.2.0",
@@ -3968,11 +3968,11 @@
       }
     },
     "node_modules/axios": {
-      "version": "1.6.7",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.7.tgz",
-      "integrity": "sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==",
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz",
+      "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==",
       "dependencies": {
-        "follow-redirects": "^1.15.4",
+        "follow-redirects": "^1.15.6",
         "form-data": "^4.0.0",
         "proxy-from-env": "^1.1.0"
       }
@@ -21489,11 +21489,11 @@
       "dev": true
     },
     "axios": {
-      "version": "1.6.7",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.7.tgz",
-      "integrity": "sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==",
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz",
+      "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==",
       "requires": {
-        "follow-redirects": "^1.15.4",
+        "follow-redirects": "^1.15.6",
         "form-data": "^4.0.0",
         "proxy-from-env": "^1.1.0"
       }
diff --git a/web-console/package.json b/web-console/package.json
index d55bb79d609..6d8c0c6448a 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -73,7 +73,7 @@
     "@druid-toolkit/visuals-react": "^0.3.3",
     "@fontsource/open-sans": "^5.0.28",
     "ace-builds": "~1.4.14",
-    "axios": "^1.6.7",
+    "axios": "^1.7.4",
     "chronoshift": "^0.10.0",
     "classnames": "^2.2.6",
     "copy-to-clipboard": "^3.2.0",

From 8181ef627a2ddbba347a30b2eee66fdb9c0535cc Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Wed, 14 Aug 2024 13:44:53 -0700
Subject: [PATCH 14/99] add useConcurrentLocks toggle (#16899)

---
 .../query-context/query-context.tsx           |   2 +
 .../__snapshots__/run-panel.spec.tsx.snap     | 176 ++++++++++++++++++
 .../run-panel/run-panel.spec.tsx              |  60 ++++++
 .../workbench-view/run-panel/run-panel.tsx    |  86 +++++----
 4 files changed, 286 insertions(+), 38 deletions(-)
 create mode 100644 web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap
 create mode 100644 web-console/src/views/workbench-view/run-panel/run-panel.spec.tsx

diff --git a/web-console/src/druid-models/query-context/query-context.tsx b/web-console/src/druid-models/query-context/query-context.tsx
index a25d268d845..4b0367c25a6 100644
--- a/web-console/src/druid-models/query-context/query-context.tsx
+++ b/web-console/src/druid-models/query-context/query-context.tsx
@@ -40,6 +40,7 @@ export interface QueryContext {
   sqlJoinAlgorithm?: SqlJoinAlgorithm;
   failOnEmptyInsert?: boolean;
   waitUntilSegmentsLoad?: boolean;
+  useConcurrentLocks?: boolean;
 
   [key: string]: any;
 }
@@ -61,6 +62,7 @@ export const DEFAULT_SERVER_QUERY_CONTEXT: QueryContext = {
   sqlJoinAlgorithm: 'broadcast',
   failOnEmptyInsert: false,
   waitUntilSegmentsLoad: false,
+  useConcurrentLocks: false,
 };
 
 export interface QueryWithContext {
diff --git a/web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap b/web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap
new file mode 100644
index 00000000000..8c7f4ec7a70
--- /dev/null
+++ b/web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap
@@ -0,0 +1,176 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`RunPanel matches snapshot on msq (auto) query 1`] = `
+<div
+  class="run-panel"
+>
+  <button
+    class="bp5-button bp5-intent-primary"
+    type="button"
+  >
+    <span
+      aria-hidden="true"
+      class="bp5-icon bp5-icon-caret-right"
+    >
+      <svg
+        data-icon="caret-right"
+        height="16"
+        role="img"
+        viewBox="0 0 16 16"
+        width="16"
+      >
+        <path
+          d="M11 8c0-.15-.07-.28-.17-.37l-4-3.5A.495.495 0 006 4.5v7a.495.495 0 00.83.37l4-3.5c.1-.09.17-.22.17-.37z"
+          fill-rule="evenodd"
+        />
+      </svg>
+    </span>
+    <span
+      class="bp5-button-text"
+    >
+      Run
+    </span>
+  </button>
+  <div
+    class="bp5-button-group"
+  >
+    <span
+      class="bp5-popover-target"
+    >
+      <button
+        aria-expanded="false"
+        aria-haspopup="menu"
+        class="bp5-button"
+        type="button"
+      >
+        <span
+          class="bp5-button-text"
+        >
+          Engine: sql-msq-task
+        </span>
+        <span
+          aria-hidden="true"
+          class="bp5-icon bp5-icon-caret-down"
+        >
+          <svg
+            data-icon="caret-down"
+            height="16"
+            role="img"
+            viewBox="0 0 16 16"
+            width="16"
+          >
+            <path
+              d="M12 6.5c0-.28-.22-.5-.5-.5h-7a.495.495 0 00-.37.83l3.5 4c.09.1.22.17.37.17s.28-.07.37-.17l3.5-4c.08-.09.13-.2.13-.33z"
+              fill-rule="evenodd"
+            />
+          </svg>
+        </span>
+      </button>
+    </span>
+    <span
+      class="max-tasks-button bp5-popover-target"
+    >
+      <button
+        aria-expanded="false"
+        aria-haspopup="menu"
+        class="bp5-button"
+        type="button"
+      >
+        <span
+          class="bp5-button-text"
+        >
+          Max tasks: 9 (full cluster capacity)
+        </span>
+        <span
+          aria-hidden="true"
+          class="bp5-icon bp5-icon-caret-down"
+        >
+          <svg
+            data-icon="caret-down"
+            height="16"
+            role="img"
+            viewBox="0 0 16 16"
+            width="16"
+          >
+            <path
+              d="M12 6.5c0-.28-.22-.5-.5-.5h-7a.495.495 0 00-.37.83l3.5 4c.09.1.22.17.37.17s.28-.07.37-.17l3.5-4c.08-.09.13-.2.13-.33z"
+              fill-rule="evenodd"
+            />
+          </svg>
+        </span>
+      </button>
+    </span>
+  </div>
+</div>
+`;
+
+exports[`RunPanel matches snapshot on native (auto) query 1`] = `
+<div
+  class="run-panel"
+>
+  <button
+    class="bp5-button bp5-intent-primary"
+    type="button"
+  >
+    <span
+      aria-hidden="true"
+      class="bp5-icon bp5-icon-caret-right"
+    >
+      <svg
+        data-icon="caret-right"
+        height="16"
+        role="img"
+        viewBox="0 0 16 16"
+        width="16"
+      >
+        <path
+          d="M11 8c0-.15-.07-.28-.17-.37l-4-3.5A.495.495 0 006 4.5v7a.495.495 0 00.83.37l4-3.5c.1-.09.17-.22.17-.37z"
+          fill-rule="evenodd"
+        />
+      </svg>
+    </span>
+    <span
+      class="bp5-button-text"
+    >
+      Run
+    </span>
+  </button>
+  <div
+    class="bp5-button-group"
+  >
+    <span
+      class="bp5-popover-target"
+    >
+      <button
+        aria-expanded="false"
+        aria-haspopup="menu"
+        class="bp5-button"
+        type="button"
+      >
+        <span
+          class="bp5-button-text"
+        >
+          Engine: auto (sql-native)
+        </span>
+        <span
+          aria-hidden="true"
+          class="bp5-icon bp5-icon-caret-down"
+        >
+          <svg
+            data-icon="caret-down"
+            height="16"
+            role="img"
+            viewBox="0 0 16 16"
+            width="16"
+          >
+            <path
+              d="M12 6.5c0-.28-.22-.5-.5-.5h-7a.495.495 0 00-.37.83l3.5 4c.09.1.22.17.37.17s.28-.07.37-.17l3.5-4c.08-.09.13-.2.13-.33z"
+              fill-rule="evenodd"
+            />
+          </svg>
+        </span>
+      </button>
+    </span>
+  </div>
+</div>
+`;
diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.spec.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.spec.tsx
new file mode 100644
index 00000000000..5416984c3db
--- /dev/null
+++ b/web-console/src/views/workbench-view/run-panel/run-panel.spec.tsx
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { render } from '@testing-library/react';
+import React from 'react';
+
+import { DEFAULT_SERVER_QUERY_CONTEXT, DRUID_ENGINES, WorkbenchQuery } from '../../../druid-models';
+
+import { RunPanel } from './run-panel';
+
+describe('RunPanel', () => {
+  it('matches snapshot on native (auto) query', () => {
+    const runPanel = (
+      <RunPanel
+        query={WorkbenchQuery.blank().changeQueryString(`SELECT * FROM wikipedia`)}
+        onQueryChange={() => {}}
+        running={false}
+        onRun={() => {}}
+        queryEngines={DRUID_ENGINES}
+        clusterCapacity={9}
+        defaultQueryContext={DEFAULT_SERVER_QUERY_CONTEXT}
+      />
+    );
+    const { container } = render(runPanel);
+    expect(container.firstChild).toMatchSnapshot();
+  });
+
+  it('matches snapshot on msq (auto) query', () => {
+    const runPanel = (
+      <RunPanel
+        query={WorkbenchQuery.blank()
+          .changeQueryString(`SELECT * FROM wikipedia`)
+          .changeEngine('sql-msq-task')}
+        onQueryChange={() => {}}
+        running={false}
+        onRun={() => {}}
+        queryEngines={DRUID_ENGINES}
+        clusterCapacity={9}
+        defaultQueryContext={DEFAULT_SERVER_QUERY_CONTEXT}
+      />
+    );
+    const { container } = render(runPanel);
+    expect(container.firstChild).toMatchSnapshot();
+  });
+});
diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
index 9ed135da354..e329f9d383b 100644
--- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx
+++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
@@ -103,7 +103,6 @@ export interface RunPanelProps
   query: WorkbenchQuery;
   onQueryChange(query: WorkbenchQuery): void;
   running: boolean;
-  small?: boolean;
   onRun(preview: boolean): void | Promise<void>;
   queryEngines: DruidEngine[];
   clusterCapacity: number | undefined;
@@ -120,7 +119,6 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
     onRun,
     moreMenu,
     running,
-    small,
     queryEngines,
     clusterCapacity,
     defaultQueryContext,
@@ -166,6 +164,11 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
     queryContext,
     defaultQueryContext,
   );
+  const useConcurrentLocks = getQueryContextKey(
+    'useConcurrentLocks',
+    queryContext,
+    defaultQueryContext,
+  );
   const finalizeAggregations = queryContext.finalizeAggregations;
   const waitUntilSegmentsLoad = queryContext.waitUntilSegmentsLoad;
   const groupByEnableMultiValueUnnesting = queryContext.groupByEnableMultiValueUnnesting;
@@ -198,7 +201,6 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
   }, [onRun]);
 
   const hotkeys = useMemo(() => {
-    if (small) return [];
     return [
       {
         allowInInput: true,
@@ -217,7 +219,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
         onKeyDown: handlePreview,
       },
     ];
-  }, [small, handleRun, handlePreview]);
+  }, [handleRun, handlePreview]);
 
   useHotkeys(hotkeys);
 
@@ -264,9 +266,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
         icon={IconNames.CARET_RIGHT}
         onClick={() => void onRun(false)}
         text="Run"
-        intent={!emptyQuery && !small ? Intent.PRIMARY : undefined}
-        small={small}
-        minimal={small}
+        intent={!emptyQuery ? Intent.PRIMARY : undefined}
       />
       {ingestMode && (
         <Button
@@ -274,11 +274,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
           icon={IconNames.EYE_OPEN}
           onClick={() => void onRun(true)}
           text="Preview"
-          small={small}
-          minimal={small}
         />
       )}
-      {!small && onQueryChange && (
+      {onQueryChange && (
         <ButtonGroup>
           <Popover
             position={Position.BOTTOM_LEFT}
@@ -366,6 +364,45 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                 )}
                 {effectiveEngine === 'sql-msq-task' ? (
                   <>
+                    <MenuItem icon={IconNames.BRING_DATA} text="INSERT / REPLACE specific context">
+                      <MenuCheckbox
+                        checked={useConcurrentLocks}
+                        text="Use concurrent locks"
+                        onChange={() =>
+                          changeQueryContext({
+                            ...queryContext,
+                            useConcurrentLocks: !useConcurrentLocks,
+                          })
+                        }
+                      />
+                      <MenuTristate
+                        icon={IconNames.DISABLE}
+                        text="Fail on empty insert"
+                        value={failOnEmptyInsert}
+                        undefinedEffectiveValue={false}
+                        onValueChange={failOnEmptyInsert =>
+                          changeQueryContext({ ...queryContext, failOnEmptyInsert })
+                        }
+                      />
+                      <MenuTristate
+                        icon={IconNames.STOPWATCH}
+                        text="Wait until segments have loaded"
+                        value={waitUntilSegmentsLoad}
+                        undefinedEffectiveValue={ingestMode}
+                        onValueChange={waitUntilSegmentsLoad =>
+                          changeQueryContext({ ...queryContext, waitUntilSegmentsLoad })
+                        }
+                      />
+                      <MenuItem
+                        icon={IconNames.TH_DERIVED}
+                        text="Edit index spec"
+                        label={summarizeIndexSpec(indexSpec)}
+                        shouldDismissPopover={false}
+                        onClick={() => {
+                          setIndexSpecDialogSpec(indexSpec || {});
+                        }}
+                      />
+                    </MenuItem>
                     <MenuItem
                       icon={IconNames.ERROR}
                       text="Max parse exceptions"
@@ -383,15 +420,6 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                         />
                       ))}
                     </MenuItem>
-                    <MenuTristate
-                      icon={IconNames.DISABLE}
-                      text="Fail on empty insert"
-                      value={failOnEmptyInsert}
-                      undefinedEffectiveValue={false}
-                      onValueChange={failOnEmptyInsert =>
-                        changeQueryContext({ ...queryContext, failOnEmptyInsert })
-                      }
-                    />
                     <MenuTristate
                       icon={IconNames.TRANSLATE}
                       text="Finalize aggregations"
@@ -401,15 +429,6 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                         changeQueryContext({ ...queryContext, finalizeAggregations })
                       }
                     />
-                    <MenuTristate
-                      icon={IconNames.STOPWATCH}
-                      text="Wait until segments have loaded"
-                      value={waitUntilSegmentsLoad}
-                      undefinedEffectiveValue={ingestMode}
-                      onValueChange={waitUntilSegmentsLoad =>
-                        changeQueryContext({ ...queryContext, waitUntilSegmentsLoad })
-                      }
-                    />
                     <MenuTristate
                       icon={IconNames.FORK}
                       text="Enable GroupBy multi-value unnesting"
@@ -477,15 +496,6 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                         })
                       }
                     />
-                    <MenuItem
-                      icon={IconNames.TH_DERIVED}
-                      text="Edit index spec"
-                      label={summarizeIndexSpec(indexSpec)}
-                      shouldDismissPopover={false}
-                      onClick={() => {
-                        setIndexSpecDialogSpec(indexSpec || {});
-                      }}
-                    />
                   </>
                 ) : (
                   <>
@@ -602,7 +612,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
       )}
       {moreMenu && (
         <Popover position={Position.BOTTOM_LEFT} content={moreMenu}>
-          <Button small={small} minimal={small} rightIcon={IconNames.MORE} />
+          <Button rightIcon={IconNames.MORE} />
         </Popover>
       )}
       {editContextDialogOpen && (

From 964cf47bb5e515a8a65969e983a80827df1aea2c Mon Sep 17 00:00:00 2001
From: Sree Charan Manamala <sree.manamala@imply.io>
Date: Thu, 15 Aug 2024 15:42:22 +0530
Subject: [PATCH 15/99] fix NPE (#16897)

---
 .../query/rowsandcols/ArrayListRowsAndColumns.java  |  3 +++
 .../calcite/tests/window/arrayConcatAgg.sqlTest     | 13 +++++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 sql/src/test/resources/calcite/tests/window/arrayConcatAgg.sqlTest

diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumns.java
index 3b5541ca5bb..883371fec65 100644
--- a/processing/src/main/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumns.java
+++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/ArrayListRowsAndColumns.java
@@ -144,6 +144,9 @@ public class ArrayListRowsAndColumns<RowType> implements AppendableRowsAndColumn
   {
     if (!rowSignature.contains(name)) {
       final Column retVal = extraColumns.get(name);
+      if (retVal == null) {
+        return null;
+      }
       if (numRows() == rows.size()) {
         return retVal;
       }
diff --git a/sql/src/test/resources/calcite/tests/window/arrayConcatAgg.sqlTest b/sql/src/test/resources/calcite/tests/window/arrayConcatAgg.sqlTest
new file mode 100644
index 00000000000..9ec451a94d9
--- /dev/null
+++ b/sql/src/test/resources/calcite/tests/window/arrayConcatAgg.sqlTest
@@ -0,0 +1,13 @@
+type: "operatorValidation"
+
+sql: |
+  SELECT
+    cityName,
+    array_concat_agg(ARRAY[cityName], 10000) over (PARTITION BY cityName)
+  FROM wikipedia
+  WHERE countryName='Austria' and cityName is not null
+  GROUP BY cityName
+
+expectedResults:
+  - ["Horsching","[\"Horsching\"]"]
+  - ["Vienna","[\"Vienna\"]"]

From 8c030feefc34ee9be689d973b8e2daf65ba2c2c7 Mon Sep 17 00:00:00 2001
From: Hugh Evans <101668034+hevansDev@users.noreply.github.com>
Date: Thu, 15 Aug 2024 17:26:34 +0100
Subject: [PATCH 16/99] Migration guide fixes (#16902)

* Fix typo in table header

* Fixed example NVL result
---
 docs/release-info/migr-ansi-sql-null.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/release-info/migr-ansi-sql-null.md b/docs/release-info/migr-ansi-sql-null.md
index 71c7f30ffb0..5e5cd4ecef0 100644
--- a/docs/release-info/migr-ansi-sql-null.md
+++ b/docs/release-info/migr-ansi-sql-null.md
@@ -184,7 +184,7 @@ PARTITIONED BY MONTH
 
 Druid ingests the data with no null values as follows:
 
-| `__time` | `string_examle` | `number_example`|
+| `__time` | `string_example` | `number_example`|
 | -- | -- | -- |
 | `2024-01-01T00:00:00.000Z`| `my_string`| 99 |
 | `2024-01-02T00:00:00.000Z`| `empty`| 0 |
@@ -376,7 +376,7 @@ Druid returns the following:
 | -- |
 | 100 |
 | 1 |
-| null |
+| 1 |
 
 ## Learn more
 

From 6cfdeb38941bfefc0032c07359495170b2fa0181 Mon Sep 17 00:00:00 2001
From: Hugh Evans <101668034+hevansDev@users.noreply.github.com>
Date: Thu, 15 Aug 2024 18:25:09 +0100
Subject: [PATCH 17/99] Added a topic listing reserved keywords (#16843)

---
 docs/querying/sql.md | 13 +++++++++++++
 website/.spelling    |  1 +
 2 files changed, 14 insertions(+)

diff --git a/docs/querying/sql.md b/docs/querying/sql.md
index 056ea9119b6..c0241bef879 100644
--- a/docs/querying/sql.md
+++ b/docs/querying/sql.md
@@ -423,3 +423,16 @@ SELECT count(city) from druid.table where SCALAR_IN_ARRAY(city, ?)
 ```
 
 sample java code using dynamic parameters is provided [here](../api-reference/sql-jdbc.md#dynamic-parameters).
+
+## Reserved keywords
+
+Druid SQL reserves certain keywords which are used in its query language. Apache Druid inherits all of the reserved keywords from [Apache Calcite](https://calcite.apache.org/docs/reference.html#keywords). In addition to these, the following reserved keywords are unique to Apache Druid:
+
+* **CLUSTERED**
+* **PARTITIONED**
+
+To use the reserved keywords in queries, enclose them in double quotation marks. For example, the reserved keyword **PARTITIONED** can be used in a query if and only if it is correctly quoted:
+
+```sql
+SELECT "PARTITIONED" from druid.table
+```
\ No newline at end of file
diff --git a/website/.spelling b/website/.spelling
index 1ac39471571..3e9827e82be 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -324,6 +324,7 @@ deserializes
 downtimes
 druid
 druid–kubernetes-extensions
+druid.table
 e.g.
 encodings
 endian

From e91f680d508e67f235cd16b6c886cd1275e92672 Mon Sep 17 00:00:00 2001
From: Hugh Evans <101668034+hevansDev@users.noreply.github.com>
Date: Thu, 15 Aug 2024 19:54:34 +0100
Subject: [PATCH 18/99] Removed deprecated deep storage properties (#16904)

---
 docs/development/extensions-core/s3.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/development/extensions-core/s3.md b/docs/development/extensions-core/s3.md
index ab8745f6e3b..9690c27e4ca 100644
--- a/docs/development/extensions-core/s3.md
+++ b/docs/development/extensions-core/s3.md
@@ -55,8 +55,6 @@ To use S3 for Deep Storage, you must supply [connection information](#configurat
 |`druid.storage.bucket`|Bucket to store in.|Must be set.|
 |`druid.storage.baseKey`|A prefix string that will be prepended to the object names for the segments published to S3 deep storage|Must be set.|
 |`druid.storage.type`|Global deep storage provider. Must be set to `s3` to make use of this extension.|Must be set (likely `s3`).|
-|`druid.storage.archiveBucket`|S3 bucket name for archiving when running the *archive task*.|none|
-|`druid.storage.archiveBaseKey`|S3 object key prefix for archiving.|none|
 |`druid.storage.disableAcl`|Boolean flag for how object permissions are handled. To use ACLs, set this property to `false`. To use Object Ownership, set it to `true`. The permission requirements for ACLs and Object Ownership are different. For more information, see [S3 permissions settings](#s3-permissions-settings).|false|
 |`druid.storage.useS3aSchema`|If true, use the "s3a" filesystem when using Hadoop-based ingestion. If false, the "s3n" filesystem will be used. Only affects Hadoop-based ingestion.|false|
 

From 5b94839d9d9185faa3a14feb3748e62e16c77c50 Mon Sep 17 00:00:00 2001
From: Edgar Melendrez <evmelendrez@gmail.com>
Date: Thu, 15 Aug 2024 19:15:30 -0700
Subject: [PATCH 19/99] [Docs] Batch08: adding examples to string functions
 (#16871)

* batch08 completed

* reviewing batch08

* apply corrections suggestions by @FrankChen021
---
 docs/querying/sql-functions.md | 268 ++++++++++++++++++++++++++++-----
 docs/querying/sql-scalar.md    |   6 +-
 2 files changed, 235 insertions(+), 39 deletions(-)

diff --git a/docs/querying/sql-functions.md b/docs/querying/sql-functions.md
index 4325537a627..2a814337176 100644
--- a/docs/querying/sql-functions.md
+++ b/docs/querying/sql-functions.md
@@ -597,11 +597,30 @@ Returns true if the expression is contained in a Base64-serialized Bloom filter.
 
 ## BTRIM
 
-`BTRIM(<CHARACTER>, [<CHARACTER>])`
+Trims characters from both the leading and trailing ends of an expression. Defaults `chars` to a space if none is provided.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `BTRIM(expr[, chars])`
+* **Function type:** Scalar, string
 
-Trims characters from both the leading and trailing ends of an expression.
+<details><summary>Example</summary>
+
+The following example trims the `_` characters from both ends of the string expression.
+
+```sql
+SELECT 
+  '___abc___' AS "original_string",
+  BTRIM('___abc___', '_') AS "trim_both_ends"
+```
+
+Returns the following:
+
+| `original_string` | `trim_both_ends` |
+| -- | -- | 
+| `___abc___` | `abc` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## CASE
 
@@ -1549,11 +1568,30 @@ Returns the leftmost number of characters from an expression, optionally padded
 
 ## LTRIM
 
-`LTRIM(<CHARACTER>, [<CHARACTER>])`
+Trims characters from the leading end of an expression. Defaults `chars` to a space if none is provided.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `LTRIM(expr[, chars])`
+* **Function type:** Scalar, string
 
-Trims characters from the leading end of an expression.
+<details><summary>Example</summary>
+
+The following example trims the `_` characters from the leading end of the string expression.
+
+```sql
+SELECT 
+  '___abc___' AS "original_string",
+  LTRIM('___abc___', '_') AS "trim_leading_end_of_expression"
+```
+
+Returns the following:
+
+| `original_string` | `trim_leading_end_of_expression` |
+| -- | -- | 
+| `___abc___` | `abc___` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## MAX
 
@@ -1859,28 +1897,85 @@ string may refer to capture groups using `$1`, `$2`, etc.
 
 ## REPEAT
 
-`REPEAT(<CHARACTER>, [<INTEGER>])`
+Repeats the string expression `N` times, where `N` is an integer.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `REPEAT(expr, N)`
+* **Function type:** Scalar, string
 
-Repeats the string expression an integer number of times.
+<details><summary>Example</summary>
+
+The following example returns the string expression `abc` repeated `3` times.
+
+```sql
+SELECT 
+  'abc' AS "original_string",
+  REPEAT('abc', 3) AS "with_repetition"
+```
+
+Returns the following: 
+
+| `original_string` | `with_repetition` |
+| -- | -- |
+| `abc` | `abcabcabc` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## REPLACE
 
-`REPLACE(expr, pattern, replacement)`
+Replaces instances of a substring with a replacement string in the given expression.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `REPLACE(expr, substring, replacement)`
+* **Function type:** Scalar, string 
 
-Replaces a pattern with another string in the given expression.
+<details><summary>Example</summary>
+
+The following example replaces instances of the substring `abc` with `XYZ`.
+
+```sql
+SELECT 
+  'abc 123 abc 123' AS "original_string",
+   REPLACE('abc 123 abc 123', 'abc', 'XYZ') AS "modified_string"
+```
+
+Returns the following:
+
+| `original_string` | `modified_string` | 
+| -- | -- |
+| `abc 123 abc 123` | `XYZ 123 XYZ 123` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## REVERSE
 
-`REVERSE(expr)`
-
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
-
 Reverses the given expression.
 
+* **Syntax:** `REVERSE(expr)`
+* **Function type:** Scalar, string
+
+<details><summary>Example</summary>
+
+The following example reverses the string expression `abc`.
+
+```sql
+SELECT 
+  'abc' AS "original_string",
+  REVERSE('abc') AS "reversal"
+```
+
+Returns the following:
+
+| `original_string` | `reversal` |
+| -- | -- |
+| `abc` | `cba` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
+
 ## RIGHT
 
 Returns the `N` rightmost characters of an expression, where `N` is an integer value.
@@ -1954,11 +2049,30 @@ Returns the rightmost number of characters from an expression, optionally padded
 
 ## RTRIM
 
-`RTRIM(<CHARACTER>, [<CHARACTER>])`
+Trims characters from the trailing end of an expression. Defaults `chars` to a space if none is provided.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `RTRIM(expr[, chars])`
+* **Function type:** Scalar, string
 
-Trims characters from the trailing end of an expression.
+<details><summary>Example</summary>
+
+The following example trims the `_` characters from the trailing end of the string expression.
+
+```sql
+SELECT 
+  '___abc___' AS "original_string",
+  RTRIM('___abc___', '_') AS "trim_end"
+```
+
+Returns the following:
+
+| `original_string` | `trim_end` |
+| -- | -- | 
+| `___abc___` | `___abc` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## SAFE_DIVIDE
 
@@ -2074,11 +2188,33 @@ Splits `str1` into an array on the delimiter specified by `str2`, which is a reg
 
 ## STRING_FORMAT
 
-`STRING_FORMAT(pattern[, args...])`
+Returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#format-java.lang.String-java.lang.Object...-).
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `STRING_FORMAT(pattern[, args...])`
+* **Function type:** Scalar, string 
 
-Returns a string formatted in accordance to Java's String.format method.
+<details><summary>Example</summary>
+
+The following example uses Java String format to pass in `Flight_Number_Reporting_Airline` and `origin_airport` columns, from the `flight-carriers` datasource, as arguments into the string. 
+
+```sql
+SELECT 
+  "Flight_Number_Reporting_Airline" AS "flight_number",
+  "Origin" AS "origin_airport",
+  STRING_FORMAT('Flight No.%d departing from %s', "Flight_Number_Reporting_Airline", "Origin") AS "departure_announcement"
+FROM "flight-carriers"
+LIMIT 1
+```
+
+Returns the following:
+
+| `flight_number` | `origin_airport` | `departure_announcement` |
+| -- | -- | -- |
+| `314` | `SJU` | `Flight No.314 departing from SJU` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## STRING_TO_MV
 
@@ -2099,27 +2235,69 @@ Alias for [`LENGTH`](#length).
 
 ## STRPOS
 
-`STRPOS(<CHARACTER>, <CHARACTER>)`
+Returns the one-based index position of a substring within an expression. If `substring` is not found, returns 0.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `STRPOS(expr, substring)`
+* **Function type:** Scalar, string
 
-Returns the one-based index position of a substring within an expression.
+<details><summary>Example</summary>
+
+The following example returns the one-based index position of `World`.
+
+```sql
+SELECT 
+  'Hello World!' AS "original_string",
+  STRPOS('Hello World!', 'World') AS "index"
+```
+
+Returns the following:
+
+| `original_string` | `index` |
+| -- | -- |
+| `Hello World!` | `7` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## SUBSTR
 
-`SUBSTR(<CHARACTER>, <INTEGER>, [<INTEGER>])`
-
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
-
 Alias for [`SUBSTRING`](#substring).
 
+* **Syntax:** `SUBSTR(expr, index[, length])`
+* **Function type:** Scalar, string
+
+[Learn more](sql-scalar.md#string-functions)
+
+
 ## SUBSTRING
 
-`SUBSTRING(<CHARACTER>, <INTEGER>, [<INTEGER>])`
+Returns a substring of the expression starting at a given one-based index. If `length` is omitted, extracts characters to the end of the string, otherwise returns a substring of `length` characters.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `SUBSTRING(expr, index[, length])`
+* **Function type:** Scalar, string
 
-Returns a substring of the expression starting at a given one-based index.
+<details><summary>Example</summary>
+
+The following example extracts a substring from the string expression `abcdefghi` of length `3` starting at index `4`
+
+```sql
+SELECT 
+  'abcdefghi' AS "original_string",
+  SUBSTRING('abcdefghi', 4, 3) AS "substring"
+```
+
+Returns the following:
+
+| `original_string` | `substring` |
+| -- | -- |
+| `abcdefghi` | `def` |
+
+</details>
+
+
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## SUM
 
@@ -2322,17 +2500,35 @@ Takes the difference between two timestamps, returning the results in the given
 **Function type:** [JSON](sql-json-functions.md)
 
 `TO_JSON_STRING(expr)`
-
 Serializes `expr` into a JSON string.
 
 
 ## TRIM
 
-`TRIM([BOTH|LEADING|TRAILING] [<chars> FROM] expr)`
+Trims the leading and/or trailing characters of an expression. Defaults `chars` to a space if none is provided. Defaults to `BOTH` if no directional argument is provided.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `TRIM([BOTH|LEADING|TRAILING] [chars FROM] expr)`
+* **Function type:** Scalar, string
 
-Trims the leading or trailing characters of an expression.
+<details><summary>Example</summary>
+
+The following example trims `_` characters from both ends of the string expression.
+
+```sql
+SELECT 
+  '___abc___' AS "original_string",
+  TRIM( BOTH '_' FROM '___abc___') AS "trim_expression"
+```
+
+Returns the following:
+
+| `original_string` | `trim_expression` |
+| -- | -- |
+| `___abc___` | `abc` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## TRUNC
 
diff --git a/docs/querying/sql-scalar.md b/docs/querying/sql-scalar.md
index fdc4a7424d1..662c9dc9ed6 100644
--- a/docs/querying/sql-scalar.md
+++ b/docs/querying/sql-scalar.md
@@ -112,12 +112,12 @@ String functions accept strings and return a type appropriate to the function.
 |`REGEXP_EXTRACT(expr, pattern[, index])`|Apply regular expression `pattern` to `expr` and extract a capture group or `NULL` if there is no match. If `index` is unspecified or zero, returns the first substring that matched the pattern. The pattern may match anywhere inside `expr`. To match the entire string, use the `^` and `$` markers at the start and end of your pattern. When `druid.generic.useDefaultValueForNull = true`, it is not possible to differentiate an empty-string match from a non-match (both return `NULL`).|
 |`REGEXP_LIKE(expr, pattern)`|Returns whether `expr` matches regular expression `pattern`. The pattern may match anywhere inside `expr`; if you want to match the entire string instead, use the `^` and `$` markers at the start and end of your pattern. Similar to [`LIKE`](sql-operators.md#logical-operators), but uses regexps instead of LIKE patterns. Especially useful in WHERE clauses.|
 |`REGEXP_REPLACE(expr, pattern, replacement)`|Replaces all occurrences of regular expression `pattern` within `expr` with `replacement`. The replacement string may refer to capture groups using `$1`, `$2`, etc. The pattern may match anywhere inside `expr`; if you want to match the entire string instead, use the `^` and `$` markers at the start and end of your pattern.|
-|`REPLACE(expr, pattern, replacement)`|Replaces pattern with replacement in `expr`, and returns the result.|
+|`REPLACE(expr, substring, replacement)`|Replaces instances of `substring` in `expr` with `replacement` and returns the result.|
 |`REPEAT(expr, N)`|Repeats `expr` `N` times.|
 |`REVERSE(expr)`|Reverses `expr`.|
 |`STRING_FORMAT(pattern[, args...])`|Returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#format-java.lang.String-java.lang.Object...-).|
-|`STRPOS(haystack, needle)`|Returns the index of `needle` within `haystack`, with indexes starting from 1. If `needle` is not found, returns 0.|
-|`SUBSTRING(expr, index[, length])`|Returns a substring of `expr` starting at `index`, with a max `length`, both measured in UTF-16 code units.|
+|`STRPOS(expr, substring)`|Returns the index of `substring` within `expr`, with indexes starting from 1. If `substring` is not found, returns 0.|
+|`SUBSTRING(expr, index[, length])`|Returns a substring of `expr` starting at a given one-based index. If `length` is omitted, extracts characters to the end of the string, otherwise returns a substring of `length` UTF-16 characters.|
 |`SUBSTR(expr, index[, length])`|Alias for `SUBSTRING`.|
 |`TRIM([BOTH `<code>&#124;</code>` LEADING `<code>&#124;</code>` TRAILING] [chars FROM] expr)`|Returns `expr` with characters removed from the leading, trailing, or both ends of `expr` if they are in `chars`. If `chars` is not provided, it defaults to `''` (a space). If the directional argument is not provided, it defaults to `BOTH`.|
 |`BTRIM(expr[, chars])`|Alternate form of `TRIM(BOTH chars FROM expr)`.|

From e37fe93f093a2d2991d39ff3245fc1d07f1e85d4 Mon Sep 17 00:00:00 2001
From: Vishesh Garg <gargvishesh@gmail.com>
Date: Fri, 16 Aug 2024 15:24:49 +0530
Subject: [PATCH 20/99] Add support for a custom `DimensionSchema` in
 `DataSourceMSQDestination` (#16864)

This PR adds support for passing in a custom DimensionSchema map to MSQ query destination of type DataSourceMSQDestination
---
 .../apache/druid/msq/exec/ControllerImpl.java | 49 ++++++++++-----
 .../msq/indexing/MSQCompactionRunner.java     | 10 ++-
 .../apache/druid/msq/indexing/MSQSpec.java    |  3 +-
 .../destination/DataSourceMSQDestination.java | 26 +++++++-
 .../druid/msq/sql/MSQTaskQueryMaker.java      |  3 +-
 .../msq/exec/MSQParseExceptionsTest.java      |  4 +-
 .../msq/indexing/MSQCompactionRunnerTest.java | 17 +++--
 .../msq/indexing/MSQControllerTaskTest.java   |  3 +-
 .../DataSourceMSQDestinationTest.java         | 26 +++++++-
 .../resources/SqlStatementResourceTest.java   |  1 +
 .../util/SqlStatementResourceHelperTest.java  |  1 +
 .../duty/ITAutoCompactionTest.java            | 62 +++++++++++++++++++
 12 files changed, 167 insertions(+), 38 deletions(-)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
index 839839db4e4..278a85685dd 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
@@ -1950,7 +1950,8 @@ public class ControllerImpl implements Controller
             destination.getSegmentSortOrder(),
             columnMappings,
             isRollupQuery,
-            querySpec.getQuery()
+            querySpec.getQuery(),
+            destination.getDimensionToSchemaMap()
         );
 
     return new DataSchema(
@@ -2122,13 +2123,34 @@ public class ControllerImpl implements Controller
     return new StringTuple(array);
   }
 
+  private static DimensionSchema getDimensionSchema(
+      final String outputColumnName,
+      @Nullable final ColumnType queryType,
+      QueryContext context,
+      @Nullable Map<String, DimensionSchema> dimensionToSchemaMap
+  )
+  {
+    if (dimensionToSchemaMap != null && dimensionToSchemaMap.containsKey(outputColumnName)) {
+      return dimensionToSchemaMap.get(outputColumnName);
+    }
+    // In case of ingestion, or when metrics are converted to dimensions when compaction is performed without rollup,
+    // we won't have an entry in the map. For those cases, use the default config.
+    return DimensionSchemaUtils.createDimensionSchema(
+        outputColumnName,
+        queryType,
+        MultiStageQueryContext.useAutoColumnSchemas(context),
+        MultiStageQueryContext.getArrayIngestMode(context)
+    );
+  }
+
   private static Pair<List<DimensionSchema>, List<AggregatorFactory>> makeDimensionsAndAggregatorsForIngestion(
       final RowSignature querySignature,
       final ClusterBy queryClusterBy,
       final List<String> segmentSortOrder,
       final ColumnMappings columnMappings,
       final boolean isRollupQuery,
-      final Query<?> query
+      final Query<?> query,
+      @Nullable final Map<String, DimensionSchema> dimensionToSchemaMap
   )
   {
     // Log a warning unconditionally if arrayIngestMode is MVD, since the behaviour is incorrect, and is subject to
@@ -2214,18 +2236,14 @@ public class ControllerImpl implements Controller
               outputColumnAggregatorFactories,
               outputColumnName,
               type,
-              query.context()
+              query.context(),
+              dimensionToSchemaMap
           );
         } else {
           // complex columns only
           if (DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.containsKey(type.getComplexTypeName())) {
             dimensions.add(
-                DimensionSchemaUtils.createDimensionSchema(
-                    outputColumnName,
-                    type,
-                    MultiStageQueryContext.useAutoColumnSchemas(query.context()),
-                    MultiStageQueryContext.getArrayIngestMode(query.context())
-                )
+                getDimensionSchema(outputColumnName, type, query.context(), dimensionToSchemaMap)
             );
           } else if (!isRollupQuery) {
             aggregators.add(new PassthroughAggregatorFactory(outputColumnName, type.getComplexTypeName()));
@@ -2236,7 +2254,8 @@ public class ControllerImpl implements Controller
                 outputColumnAggregatorFactories,
                 outputColumnName,
                 type,
-                query.context()
+                query.context(),
+                dimensionToSchemaMap
             );
           }
         }
@@ -2263,19 +2282,15 @@ public class ControllerImpl implements Controller
       Map<String, AggregatorFactory> outputColumnAggregatorFactories,
       String outputColumn,
       ColumnType type,
-      QueryContext context
+      QueryContext context,
+      Map<String, DimensionSchema> dimensionToSchemaMap
   )
   {
     if (outputColumnAggregatorFactories.containsKey(outputColumn)) {
       aggregators.add(outputColumnAggregatorFactories.get(outputColumn));
     } else {
       dimensions.add(
-          DimensionSchemaUtils.createDimensionSchema(
-              outputColumn,
-              type,
-              MultiStageQueryContext.useAutoColumnSchemas(context),
-              MultiStageQueryContext.getArrayIngestMode(context)
-          )
+          getDimensionSchema(outputColumn, type, context, dimensionToSchemaMap)
       );
     }
   }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
index efc2cbb2afb..3abf4b59c7d 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
@@ -81,6 +81,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 public class MSQCompactionRunner implements CompactionRunner
@@ -237,7 +238,11 @@ public class MSQCompactionRunner implements CompactionRunner
         dataSchema.getDataSource(),
         dataSchema.getGranularitySpec().getSegmentGranularity(),
         null,
-        ImmutableList.of(replaceInterval)
+        ImmutableList.of(replaceInterval),
+        dataSchema.getDimensionsSpec()
+                  .getDimensions()
+                  .stream()
+                  .collect(Collectors.toMap(DimensionSchema::getName, Function.identity()))
     );
   }
 
@@ -494,9 +499,10 @@ public class MSQCompactionRunner implements CompactionRunner
     // Used for writing the data schema during segment generation phase.
     context.putIfAbsent(MultiStageQueryContext.CTX_FINALIZE_AGGREGATIONS, false);
     // Add appropriate finalization to native query context i.e. for the GroupBy query
-    context.put(QueryContexts.FINALIZE_KEY, false);
+    context.putIfAbsent(QueryContexts.FINALIZE_KEY, false);
     // Only scalar or array-type dimensions are allowed as grouping keys.
     context.putIfAbsent(GroupByQueryConfig.CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING, false);
+    context.putIfAbsent(MultiStageQueryContext.CTX_ARRAY_INGEST_MODE, "array");
     return context;
   }
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQSpec.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQSpec.java
index 065471d32ba..4bb4e32754f 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQSpec.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQSpec.java
@@ -28,7 +28,6 @@ import org.apache.druid.msq.kernel.WorkerAssignmentStrategy;
 import org.apache.druid.query.Query;
 import org.apache.druid.sql.calcite.planner.ColumnMappings;
 
-import javax.annotation.Nullable;
 import java.util.Map;
 import java.util.Objects;
 
@@ -43,7 +42,7 @@ public class MSQSpec
   @JsonCreator
   public MSQSpec(
       @JsonProperty("query") Query<?> query,
-      @JsonProperty("columnMappings") @Nullable ColumnMappings columnMappings,
+      @JsonProperty("columnMappings") ColumnMappings columnMappings,
       @JsonProperty("destination") MSQDestination destination,
       @JsonProperty("assignmentStrategy") WorkerAssignmentStrategy assignmentStrategy,
       @JsonProperty("tuningConfig") MSQTuningConfig tuningConfig
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
index ea3072bfe45..74be1329467 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
+import org.apache.druid.data.input.impl.DimensionSchema;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularity;
@@ -35,6 +36,7 @@ import org.joda.time.Interval;
 import javax.annotation.Nullable;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -49,18 +51,23 @@ public class DataSourceMSQDestination implements MSQDestination
   @Nullable
   private final List<Interval> replaceTimeChunks;
 
+  @Nullable
+  private final Map<String, DimensionSchema> dimensionToSchemaMap;
+
   @JsonCreator
   public DataSourceMSQDestination(
       @JsonProperty("dataSource") String dataSource,
       @JsonProperty("segmentGranularity") Granularity segmentGranularity,
       @JsonProperty("segmentSortOrder") @Nullable List<String> segmentSortOrder,
-      @JsonProperty("replaceTimeChunks") @Nullable List<Interval> replaceTimeChunks
+      @JsonProperty("replaceTimeChunks") @Nullable List<Interval> replaceTimeChunks,
+      @JsonProperty("dimensionToSchemaMap") @Nullable Map<String, DimensionSchema> dimensionToSchemaMap
   )
   {
     this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource");
     this.segmentGranularity = Preconditions.checkNotNull(segmentGranularity, "segmentGranularity");
     this.segmentSortOrder = segmentSortOrder != null ? segmentSortOrder : Collections.emptyList();
     this.replaceTimeChunks = replaceTimeChunks;
+    this.dimensionToSchemaMap = dimensionToSchemaMap;
 
     if (replaceTimeChunks != null) {
       // Verify that if replaceTimeChunks is provided, it is nonempty.
@@ -125,6 +132,17 @@ public class DataSourceMSQDestination implements MSQDestination
     return replaceTimeChunks;
   }
 
+  /**
+   * Returns the map of dimension name to its schema.
+   */
+  @Nullable
+  @JsonProperty
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  public Map<String, DimensionSchema> getDimensionToSchemaMap()
+  {
+    return dimensionToSchemaMap;
+  }
+
   /**
    * Whether this object is in replace-existing-time-chunks mode.
    */
@@ -158,13 +176,14 @@ public class DataSourceMSQDestination implements MSQDestination
     return Objects.equals(dataSource, that.dataSource)
            && Objects.equals(segmentGranularity, that.segmentGranularity)
            && Objects.equals(segmentSortOrder, that.segmentSortOrder)
-           && Objects.equals(replaceTimeChunks, that.replaceTimeChunks);
+           && Objects.equals(replaceTimeChunks, that.replaceTimeChunks)
+           && Objects.equals(dimensionToSchemaMap, that.dimensionToSchemaMap);
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(dataSource, segmentGranularity, segmentSortOrder, replaceTimeChunks);
+    return Objects.hash(dataSource, segmentGranularity, segmentSortOrder, replaceTimeChunks, dimensionToSchemaMap);
   }
 
   @Override
@@ -175,6 +194,7 @@ public class DataSourceMSQDestination implements MSQDestination
            ", segmentGranularity=" + segmentGranularity +
            ", segmentSortOrder=" + segmentSortOrder +
            ", replaceTimeChunks=" + replaceTimeChunks +
+           ", dimensionToSchemaMap=" + dimensionToSchemaMap +
            '}';
   }
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
index c6396c0b306..7af34a1eb55 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
@@ -246,7 +246,8 @@ public class MSQTaskQueryMaker implements QueryMaker
           targetDataSource.getDestinationName(),
           segmentGranularityObject,
           segmentSortOrder,
-          replaceTimeChunks
+          replaceTimeChunks,
+          null
       );
       MultiStageQueryContext.validateAndGetTaskLockType(sqlQueryContext,
                                                         dataSourceMSQDestination.isReplaceTimeChunks());
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java
index 330f1cdbbe6..bc8d517ffba 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java
@@ -225,7 +225,7 @@ public class MSQParseExceptionsTest extends MSQTestBase
                         new ColumnMapping("v1", "agent_category")
                     )
                 ))
-                .destination(new DataSourceMSQDestination("foo1", Granularities.ALL, null, null))
+                .destination(new DataSourceMSQDestination("foo1", Granularities.ALL, null, null, null))
                 .tuningConfig(MSQTuningConfig.defaultConfig())
                 .build())
         .setQueryContext(DEFAULT_MSQ_CONTEXT)
@@ -318,7 +318,7 @@ public class MSQParseExceptionsTest extends MSQTestBase
                         new ColumnMapping("agent_category", "agent_category")
                     )
                 ))
-                .destination(new DataSourceMSQDestination("foo1", Granularities.ALL, null, null))
+                .destination(new DataSourceMSQDestination("foo1", Granularities.ALL, null, null, null))
                 .tuningConfig(MSQTuningConfig.defaultConfig())
                 .build())
         .setQueryContext(runtimeContext)
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
index d868ddf20e5..9c4bb91637c 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
@@ -74,6 +74,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 public class MSQCompactionRunnerTest
@@ -87,13 +88,9 @@ public class MSQCompactionRunnerTest
   private static final GranularityType QUERY_GRANULARITY = GranularityType.HOUR;
   private static List<String> PARTITION_DIMENSIONS;
 
-  private static final StringDimensionSchema DIM1 = new StringDimensionSchema(
-      "string_dim",
-      null,
-      null
-  );
-  private static final LongDimensionSchema LONG_DIMENSION_SCHEMA = new LongDimensionSchema("long_dim");
-  private static final List<DimensionSchema> DIMENSIONS = ImmutableList.of(DIM1, LONG_DIMENSION_SCHEMA);
+  private static final StringDimensionSchema STRING_DIMENSION = new StringDimensionSchema("string_dim", null, null);
+  private static final LongDimensionSchema LONG_DIMENSION = new LongDimensionSchema("long_dim");
+  private static final List<DimensionSchema> DIMENSIONS = ImmutableList.of(STRING_DIMENSION, LONG_DIMENSION);
   private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
   private static final AggregatorFactory AGG1 = new CountAggregatorFactory("agg_0");
   private static final AggregatorFactory AGG2 = new LongSumAggregatorFactory("sum_added", "sum_added");
@@ -291,7 +288,8 @@ public class MSQCompactionRunnerTest
             DATA_SOURCE,
             SEGMENT_GRANULARITY.getDefaultGranularity(),
             null,
-            Collections.singletonList(COMPACTION_INTERVAL)
+            Collections.singletonList(COMPACTION_INTERVAL),
+            DIMENSIONS.stream().collect(Collectors.toMap(DimensionSchema::getName, Function.identity()))
         ),
         actualMSQSpec.getDestination()
     );
@@ -360,7 +358,8 @@ public class MSQCompactionRunnerTest
             DATA_SOURCE,
             SEGMENT_GRANULARITY.getDefaultGranularity(),
             null,
-            Collections.singletonList(COMPACTION_INTERVAL)
+            Collections.singletonList(COMPACTION_INTERVAL),
+            DIMENSIONS.stream().collect(Collectors.toMap(DimensionSchema::getName, Function.identity()))
         ),
         actualMSQSpec.getDestination()
     );
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java
index 9df6c38f30e..e969e209387 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java
@@ -58,7 +58,8 @@ public class MSQControllerTaskTest
           "target",
           Granularities.DAY,
           null,
-          INTERVALS
+          INTERVALS,
+          null
       ))
       .query(new Druids.ScanQueryBuilder()
                  .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
index 6d3a5ebfd9b..242c00213e2 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
@@ -20,9 +20,14 @@
 package org.apache.druid.msq.indexing.destination;
 
 
+import com.google.common.collect.ImmutableMap;
 import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.data.input.impl.DimensionSchema;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
 import org.junit.Test;
 
+import java.util.Map;
+
 public class DataSourceMSQDestinationTest
 {
 
@@ -30,7 +35,26 @@ public class DataSourceMSQDestinationTest
   public void testEquals()
   {
     EqualsVerifier.forClass(DataSourceMSQDestination.class)
-                  .withNonnullFields("dataSource", "segmentGranularity", "segmentSortOrder")
+                  .withNonnullFields("dataSource", "segmentGranularity", "segmentSortOrder", "dimensionToSchemaMap")
+                  .withPrefabValues(
+                      Map.class,
+                      ImmutableMap.of(
+                          "language",
+                          new StringDimensionSchema(
+                              "language",
+                              DimensionSchema.MultiValueHandling.SORTED_ARRAY,
+                              false
+                          )
+                      ),
+                      ImmutableMap.of(
+                          "region",
+                          new StringDimensionSchema(
+                              "region",
+                              DimensionSchema.MultiValueHandling.SORTED_ARRAY,
+                              false
+                          )
+                      )
+                  )
                   .usingGetClass()
                   .verify();
   }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java
index 4ea2993050e..2a753e21d16 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java
@@ -199,6 +199,7 @@ public class SqlStatementResourceTest extends MSQTestBase
                  "test",
                  Granularities.DAY,
                  null,
+                 null,
                  null
              ))
              .tuningConfig(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/SqlStatementResourceHelperTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/SqlStatementResourceHelperTest.java
index 1966d1e5b10..58856adf366 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/SqlStatementResourceHelperTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/SqlStatementResourceHelperTest.java
@@ -375,6 +375,7 @@ public class SqlStatementResourceHelperTest
             "test",
             Granularities.DAY,
             null,
+            null,
             null
         )
     );
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
index 24ede82d0de..31a6bccffc7 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
@@ -25,7 +25,9 @@ import com.google.inject.Inject;
 import org.apache.commons.io.IOUtils;
 import org.apache.datasketches.hll.TgtHllType;
 import org.apache.druid.data.input.MaxSizeSplitHintSpec;
+import org.apache.druid.data.input.impl.DimensionSchema;
 import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
 import org.apache.druid.indexer.CompactionEngine;
 import org.apache.druid.indexer.TaskState;
 import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
@@ -52,6 +54,8 @@ import org.apache.druid.query.aggregation.datasketches.hll.HllSketchBuildAggrega
 import org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchAggregatorFactory;
 import org.apache.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory;
 import org.apache.druid.query.filter.SelectorDimFilter;
+import org.apache.druid.segment.AutoTypeColumnSchema;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
@@ -515,6 +519,42 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
     }
   }
 
+  @Test(dataProvider = "engine")
+  public void testAutoCompactionPreservesCreateBitmapIndexInDimensionSchema(CompactionEngine engine) throws Exception
+  {
+    loadData(INDEX_TASK);
+    try (final Closeable ignored = unloader(fullDatasourceName)) {
+      final List<String> intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName);
+      intervalsBeforeCompaction.sort(null);
+      // 4 segments across 2 days (4 total)...
+      verifySegmentsCount(4);
+      verifyQuery(INDEX_QUERIES_RESOURCE);
+
+      LOG.info("Auto compaction test with YEAR segment granularity, dropExisting is true");
+      Granularity newSegmentGranularity = Granularities.YEAR;
+
+      List<DimensionSchema> dimensionSchemas = ImmutableList.of(
+          new StringDimensionSchema("language", DimensionSchema.MultiValueHandling.SORTED_ARRAY, false),
+          new AutoTypeColumnSchema("deleted", ColumnType.DOUBLE)
+      );
+
+      submitCompactionConfig(
+          MAX_ROWS_PER_SEGMENT_COMPACTED,
+          NO_SKIP_OFFSET,
+          new UserCompactionTaskGranularityConfig(newSegmentGranularity, null, true),
+          new UserCompactionTaskDimensionsConfig(dimensionSchemas),
+          null,
+          new AggregatorFactory[] {new LongSumAggregatorFactory("added", "added")},
+          true,
+          engine
+      );
+      //...compacted into 1 segment for the entire year.
+      forceTriggerAutoCompaction(1);
+      verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED);
+      verifySegmentsCompactedDimensionSchema(dimensionSchemas);
+    }
+  }
+
   @Test
   public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception
   {
@@ -1941,6 +1981,28 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
       Assert.assertNotNull(compactedSegment.getLastCompactionState().getPartitionsSpec());
       Assert.assertEquals(compactedSegment.getLastCompactionState().getPartitionsSpec(), partitionsSpec);
     }
+
+  }
+
+  private void verifySegmentsCompactedDimensionSchema(List<DimensionSchema> dimensionSchemas)
+  {
+    List<DataSegment> segments = coordinator.getFullSegmentsMetadata(fullDatasourceName);
+    List<DataSegment> foundCompactedSegments = new ArrayList<>();
+    for (DataSegment segment : segments) {
+      if (segment.getLastCompactionState() != null) {
+        foundCompactedSegments.add(segment);
+      }
+    }
+    for (DataSegment compactedSegment : foundCompactedSegments) {
+      MatcherAssert.assertThat(
+          dimensionSchemas,
+          Matchers.containsInAnyOrder(
+              compactedSegment.getLastCompactionState()
+                              .getDimensionsSpec()
+                              .getDimensions()
+                              .toArray(new DimensionSchema[0]))
+      );
+    }
   }
 
   private void updateCompactionTaskSlot(double compactionTaskSlotRatio, int maxCompactionTaskSlots, Boolean useAutoScaleSlots) throws Exception

From 4283b270e3ab14d99ba2c1179328365b50355f78 Mon Sep 17 00:00:00 2001
From: Clint Wylie <cwylie@apache.org>
Date: Fri, 16 Aug 2024 11:34:10 -0700
Subject: [PATCH 21/99] rework cursor creation (#16533)

changes:
* Added `CursorBuildSpec` which captures all of the 'interesting' stuff that goes into producing a cursor as a replacement for the method arguments of `CursorFactory.canVectorize`, `CursorFactory.makeCursor`, and `CursorFactory.makeVectorCursor`
* added new interface `CursorHolder` and new interface `CursorHolderFactory` as a replacement for `CursorFactory`, with method `makeCursorHolder`, which takes a `CursorBuildSpec` as an argument and replaces `CursorFactory.canVectorize`, `CursorFactory.makeCursor`, and `CursorFactory.makeVectorCursor`
* `CursorFactory.makeCursors` previously returned a `Sequence<Cursor>` corresponding to the query granularity buckets, with a separate `Cursor` per bucket. `CursorHolder.asCursor` instead returns a single `Cursor` (equivalent to 'ALL' granularity), and a new `CursorGranularizer` has been added for query engines to iterate over the cursor and divide into granularity buckets. This makes the non-vectorized engine behave the same way as the vectorized query engine (with its `VectorCursorGranularizer`), and simplifies a lot of stuff that has to read segments particularly if it does not care about bucketing the results into granularities.
* Deprecated `CursorFactory`, `CursorFactory.canVectorize`, `CursorFactory.makeCursors`, and `CursorFactory.makeVectorCursor`
* updated all `StorageAdapter` implementations to implement `makeCursorHolder`, transitioned direct `CursorFactory` implementations to instead implement `CursorMakerFactory`. `StorageAdapter` being a `CursorMakerFactory` is intended to be a transitional thing, ideally will not be released in favor of moving `CursorMakerFactory` to be fetched directly from `Segment`, however this PR was already large enough so this will be done in a follow-up.
* updated all query engines to use `makeCursorHolder`, granularity based engines to use `CursorGranularizer`.
---
 .../ExpressionAggregationBenchmark.java       |  40 +-
 .../benchmark/ExpressionFilterBenchmark.java  |  67 +-
 .../ExpressionSelectorBenchmark.java          | 687 ++++++--------
 .../ExpressionVectorSelectorBenchmark.java    |  65 +-
 .../benchmark/FilterPartitionBenchmark.java   | 159 ++--
 .../IndexedTableJoinCursorBenchmark.java      | 180 ++--
 .../benchmark/JoinAndLookupBenchmark.java     | 302 +++---
 .../IncrementalIndexReadBenchmark.java        |  75 +-
 .../benchmark/query/GroupByBenchmark.java     |  12 +-
 .../druid/benchmark/query/ScanBenchmark.java  |   3 +-
 .../benchmark/query/TimeseriesBenchmark.java  |  19 +-
 .../druid/benchmark/query/TopNBenchmark.java  |  17 +-
 .../CompressedBigDecimalFactoryTestBase.java  |   7 +-
 .../movingaverage/MovingAverageQuery.java     |   4 +-
 ...bjectVectorColumnProcessorFactoryTest.java |  31 +-
 .../msq/indexing/MSQCompactionRunner.java     |   7 +-
 .../SegmentGeneratorFrameProcessor.java       |  70 +-
 .../results/ExportResultsFrameProcessor.java  | 100 +-
 .../scan/ScanQueryFrameProcessor.java         |  83 +-
 .../druid/msq/querykit/scan/ScanQueryKit.java |   6 +-
 .../apache/druid/msq/exec/MSQArraysTest.java  |   8 +-
 .../druid/msq/exec/MSQComplexGroupByTest.java |   4 +-
 .../druid/msq/exec/MSQLoadedSegmentTests.java |   3 +-
 .../apache/druid/msq/exec/MSQSelectTest.java  |   3 +-
 .../apache/druid/msq/exec/MSQWindowTest.java  |   7 +-
 .../msq/indexing/MSQCompactionRunnerTest.java |   9 +-
 .../hadoop/DatasourceRecordReader.java        | 186 ++--
 .../indexing/input/DruidSegmentReader.java    |  29 +-
 .../common/task/CompactionTaskRunTest.java    |  60 +-
 .../indexing/common/task/IndexTaskTest.java   | 163 ++--
 .../apache/druid/frame/field/RowReader.java   |   4 +-
 .../frame/processor/FrameProcessors.java      |  20 +-
 .../apache/druid/frame/read/FrameReader.java  |  13 +-
 .../druid/frame/segment/FrameCursor.java      |  14 +-
 .../frame/segment/FrameStorageAdapter.java    |  60 +-
 .../ColumnarFrameCursorHolderFactory.java     | 190 ++++
 .../segment/columnar/FrameCursorFactory.java  | 195 ----
 .../segment/columnar/FrameQueryableIndex.java |   2 +-
 .../segment/columnar/FrameVectorCursor.java   |  19 +-
 .../frame/segment/row/FrameCursorFactory.java | 128 ---
 .../row/RowFrameCursorHolderFactory.java      | 127 +++
 .../common/granularity/Granularities.java     | 115 +++
 .../druid/math/expr/ExprMacroTable.java       |  15 +
 .../org/apache/druid/query/BaseQuery.java     |   3 +-
 .../druid/query/CursorGranularizer.java       | 175 ++++
 .../java/org/apache/druid/query/Druids.java   |   8 +-
 .../druid/query/FilteredDataSource.java       |   5 +-
 .../query/FrameBasedInlineDataSource.java     |  12 +-
 .../druid/query/IterableRowsCursorHelper.java |   2 -
 .../java/org/apache/druid/query/Order.java    |  44 +
 .../java/org/apache/druid/query/OrderBy.java  |  96 ++
 .../java/org/apache/druid/query/Queries.java  |  24 +-
 .../java/org/apache/druid/query/Query.java    |  16 +
 .../org/apache/druid/query/QueryContexts.java |   1 -
 .../apache/druid/query/QueryRunnerHelper.java |  34 -
 .../DataSourceMetadataQuery.java              |   2 +-
 .../expression/TimestampFloorExprMacro.java   |  37 +-
 .../apache/druid/query/filter/DimFilter.java  |   4 +-
 .../org/apache/druid/query/filter/Filter.java |   3 +-
 .../druid/query/filter/FilterBundle.java      |  19 +-
 .../druid/query/groupby/GroupByQuery.java     |  16 +-
 .../druid/query/groupby/GroupingEngine.java   |  57 +-
 .../epinephelinae/GroupByQueryEngine.java     | 232 +++--
 .../vector/VectorGroupByEngine.java           | 123 +--
 .../druid/query/metadata/SegmentAnalyzer.java |  61 +-
 .../metadata/SegmentMetadataQuery.java        |   2 +-
 .../query/operator/WindowOperatorQuery.java   |   8 +-
 .../LazilyDecoratedRowsAndColumns.java        |  69 +-
 .../StorageAdapterRowsAndColumns.java         |  54 +-
 .../apache/druid/query/scan/ScanQuery.java    |  87 +-
 .../druid/query/scan/ScanQueryEngine.java     | 233 ++---
 .../query/scan/ScanQueryLimitRowIterator.java |   3 +-
 .../query/scan/ScanQueryRunnerFactory.java    |   5 +-
 .../ScanResultValueTimestampComparator.java   |   3 +-
 .../query/search/CursorOnlyStrategy.java      |  96 +-
 .../search/SearchQueryQueryToolChest.java     |   2 +-
 .../query/search/UseIndexesStrategy.java      |   4 +-
 .../query/timeboundary/TimeBoundaryQuery.java |   2 +-
 .../TimeBoundaryQueryRunnerFactory.java       |  50 +-
 .../query/timeseries/TimeseriesQuery.java     |   3 +-
 .../timeseries/TimeseriesQueryEngine.java     | 189 ++--
 .../AggregateTopNMetricFirstAlgorithm.java    |  11 +-
 .../druid/query/topn/BaseTopNAlgorithm.java   |   7 +-
 .../topn/Generic1AggPooledTopNScanner.java    |   2 +
 ...Generic1AggPooledTopNScannerPrototype.java |   6 +-
 .../topn/Generic2AggPooledTopNScanner.java    |   2 +
 ...Generic2AggPooledTopNScannerPrototype.java |   6 +-
 .../query/topn/HeapBasedTopNAlgorithm.java    |   6 +-
 .../druid/query/topn/PooledTopNAlgorithm.java |  36 +-
 .../topn/TimeExtractionTopNAlgorithm.java     |  10 +-
 .../druid/query/topn/TopNAlgorithm.java       |   3 +-
 .../apache/druid/query/topn/TopNMapFn.java    |   5 +-
 .../apache/druid/query/topn/TopNParams.java   |   9 +
 .../apache/druid/query/topn/TopNQuery.java    |   5 +-
 .../druid/query/topn/TopNQueryEngine.java     | 127 ++-
 ...eNumericTopNColumnAggregatesProcessor.java |   6 +-
 .../StringTopNColumnAggregatesProcessor.java  |  19 +-
 .../types/TopNColumnAggregatesProcessor.java  |   2 +
 .../vector/VectorCursorGranularizer.java      |  70 +-
 .../druid/segment/ArrayListSegment.java       |   4 +-
 .../apache/druid/segment/BitmapOffset.java    |  38 +-
 .../java/org/apache/druid/segment/Cursor.java |  62 +-
 .../apache/druid/segment/CursorBuildSpec.java | 292 ++++++
 .../apache/druid/segment/CursorFactory.java   |  41 +-
 .../apache/druid/segment/CursorHolder.java    |  73 ++
 .../druid/segment/CursorHolderFactory.java    |  25 +
 .../org/apache/druid/segment/Cursors.java     | 105 ++
 .../druid/segment/FilteredStorageAdapter.java |  33 +-
 ...r.java => QueryableIndexCursorHolder.java} | 545 ++++++-----
 .../segment/QueryableIndexStorageAdapter.java | 115 +--
 .../apache/druid/segment/RowBasedCursor.java  |  17 +-
 .../apache/druid/segment/RowBasedSegment.java |   4 +-
 .../druid/segment/RowBasedStorageAdapter.java |  89 +-
 .../apache/druid/segment/SegmentWrangler.java |   3 +-
 .../apache/druid/segment/StorageAdapter.java  |  81 +-
 .../UnnestColumnValueSelectorCursor.java      |   9 +-
 .../druid/segment/UnnestDimensionCursor.java  |   8 +-
 .../druid/segment/UnnestStorageAdapter.java   | 126 +--
 .../apache/druid/segment/VirtualColumns.java  |  10 -
 .../org/apache/druid/segment/data/Offset.java |   6 +
 .../druid/segment/filter/AndFilter.java       |  11 +
 .../apache/druid/segment/filter/OrFilter.java |  23 +
 .../segment/incremental/IncrementalIndex.java |  10 +-
 .../IncrementalIndexCursorHolder.java         | 251 +++++
 .../IncrementalIndexRowHolder.java            |   1 +
 .../incremental/IncrementalIndexSchema.java   |  18 +-
 .../IncrementalIndexStorageAdapter.java       | 213 +----
 .../druid/segment/join/HashJoinEngine.java    |   9 +-
 .../join/HashJoinSegmentStorageAdapter.java   | 246 +++--
 .../druid/segment/join/PostJoinCursor.java    |  12 +-
 .../table/BroadcastSegmentIndexedTable.java   | 100 +-
 .../join/table/FrameBasedIndexedTable.java    |  80 +-
 .../druid/segment/vector/VectorCursor.java    |  30 +-
 .../apache/druid/utils/CollectionUtils.java   |   2 +-
 .../druid/frame/TestArrayStorageAdapter.java  |  60 +-
 .../processor/OutputChannelFactoryTest.java   |  43 +-
 .../segment/FrameStorageAdapterTest.java      |  90 +-
 .../frame/testutil/FrameSequenceBuilder.java  |  91 +-
 .../druid/frame/testutil/FrameTestUtil.java   | 125 ++-
 .../testutil/RowNumberUpdatingCursor.java     |   7 -
 .../druid/frame/write/FrameWriterTest.java    |  78 +-
 .../granularity/QueryGranularityTest.java     |  35 +
 .../druid/query/CursorGranularizerTest.java   | 255 +++++
 .../apache/druid/query/QueryContextsTest.java |  10 -
 ...TimelineMissingSegmentQueryRunnerTest.java |   1 -
 .../org/apache/druid/query/TestQuery.java     |   5 +-
 .../druid/query/groupby/GroupByQueryTest.java | 153 ++-
 .../groupby/NestedDataGroupByQueryTest.java   |  20 +-
 .../VectorGroupByEngineIteratorTest.java      |  16 +-
 .../druid/query/lookup/LookupSegmentTest.java |  58 +-
 .../metadata/SegmentMetadataQueryTest.java    |  45 +-
 .../SegmentMetadataUnionQueryTest.java        |   4 +-
 .../StorageAdapterRowsAndColumnsTest.java     |  10 +-
 .../semantic/RowsAndColumnsDecoratorTest.java |  40 +-
 .../scan/ScanQueryLimitRowIteratorTest.java   |   7 +-
 .../scan/ScanQueryResultOrderingTest.java     |  10 +-
 .../scan/ScanQueryRunnerFactoryTest.java      |  21 +-
 .../druid/query/scan/ScanQueryRunnerTest.java |   9 +-
 .../druid/query/scan/ScanQuerySpecTest.java   |  18 +-
 .../druid/query/scan/ScanQueryTest.java       |  70 +-
 ...canResultValueTimestampComparatorTest.java |   9 +-
 .../query/scan/UnnestScanQueryRunnerTest.java |   3 +-
 .../NestedDataTimeseriesQueryTest.java        |  20 +-
 .../query/timeseries/TimeseriesQueryTest.java | 103 +-
 .../topn/TopNMetricSpecOptimizationsTest.java |  26 +-
 .../druid/query/topn/TopNQueryRunnerTest.java | 159 ++++
 .../druid/query/topn/TopNQueryTest.java       |  67 +-
 .../segment/AutoTypeColumnIndexerTest.java    | 615 +++++-------
 .../org/apache/druid/segment/ListCursor.java  |   7 -
 .../NestedDataColumnIndexerV4Test.java        | 536 +++++------
 ...va => QueryableIndexCursorHolderTest.java} |  22 +-
 .../QueryableIndexStorageAdapterTest.java     |  43 +-
 .../segment/RowBasedStorageAdapterTest.java   | 893 ++++++++++--------
 .../TombstoneSegmentStorageAdapterTest.java   |  22 +-
 .../segment/UnnestStorageAdapterTest.java     | 281 +++---
 .../druid/segment/filter/BaseFilterTest.java  | 213 +++--
 .../filter/FloatAndDoubleFilteringTest.java   | 103 --
 .../segment/filter/LongFilteringTest.java     |  87 --
 .../PredicateValueMatcherFactoryTest.java     |  82 +-
 .../IncrementalIndexStorageAdapterTest.java   | 462 +++++----
 ...BaseHashJoinSegmentStorageAdapterTest.java |   3 +-
 .../HashJoinSegmentStorageAdapterTest.java    | 739 +++++----------
 .../segment/join/JoinFilterAnalyzerTest.java  | 383 +++-----
 .../druid/segment/join/JoinTestHelper.java    |  95 +-
 .../segment/join/PostJoinCursorTest.java      |  89 +-
 .../NestedFieldColumnSelectorsTest.java       |  32 +-
 .../selector/TestColumnValueSelector.java     |  19 +-
 .../virtual/ExpressionSelectorsTest.java      |  85 +-
 .../ExpressionVectorSelectorsCastTest.java    |  14 +-
 .../ExpressionVectorSelectorsTest.java        | 193 ++--
 .../CachingClusteredClientPerfTest.java       |   4 +-
 .../appenderator/StreamAppenderatorTest.java  |   3 +-
 .../apache/druid/server/TestSegmentUtils.java |  26 +-
 .../coordination/ServerManagerTest.java       |   1 -
 .../server/log/LoggingRequestLoggerTest.java  |  15 +-
 .../org/apache/druid/cli/DumpSegment.java     | 138 +--
 .../org/apache/druid/cli/DumpSegmentTest.java |  38 +
 .../druid/sql/calcite/rel/DruidQuery.java     |  10 +-
 .../CalciteCatalogIngestionDmlTest.java       |  10 +-
 .../sql/calcite/CalciteInsertDmlTest.java     |  33 +-
 .../sql/calcite/CalciteJoinQueryTest.java     |   5 +-
 .../druid/sql/calcite/CalciteQueryTest.java   |   5 +-
 .../sql/calcite/CalciteReplaceDmlTest.java    |  19 +-
 .../sql/calcite/CalciteSelectQueryTest.java   |  25 +-
 .../sql/calcite/CalciteSubqueryTest.java      |   3 +-
 205 files changed, 7976 insertions(+), 7161 deletions(-)
 create mode 100644 processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorHolderFactory.java
 delete mode 100644 processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameCursorFactory.java
 delete mode 100644 processing/src/main/java/org/apache/druid/frame/segment/row/FrameCursorFactory.java
 create mode 100644 processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorHolderFactory.java
 create mode 100644 processing/src/main/java/org/apache/druid/query/CursorGranularizer.java
 create mode 100644 processing/src/main/java/org/apache/druid/query/Order.java
 create mode 100644 processing/src/main/java/org/apache/druid/query/OrderBy.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/CursorHolder.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/CursorHolderFactory.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/Cursors.java
 rename processing/src/main/java/org/apache/druid/segment/{QueryableIndexCursorSequenceBuilder.java => QueryableIndexCursorHolder.java} (52%)
 create mode 100644 processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexCursorHolder.java
 create mode 100644 processing/src/test/java/org/apache/druid/query/CursorGranularizerTest.java
 rename processing/src/test/java/org/apache/druid/segment/{QueryableIndexCursorSequenceBuilderTest.java => QueryableIndexCursorHolderTest.java} (72%)

diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionAggregationBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionAggregationBenchmark.java
index 8afcea4a4c3..d65d13a324d 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionAggregationBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionAggregationBenchmark.java
@@ -20,11 +20,9 @@
 package org.apache.druid.benchmark;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.js.JavaScriptConfig;
 import org.apache.druid.query.aggregation.BufferAggregator;
@@ -35,9 +33,10 @@ import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.BaseFloatColumnValueSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.generator.GeneratorColumnSchema;
 import org.apache.druid.segment.generator.GeneratorSchemaInfo;
@@ -60,7 +59,6 @@ import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.nio.ByteBuffer;
-import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
@@ -165,32 +163,22 @@ public class ExpressionAggregationBenchmark
   {
     final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index);
 
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
 
-    final List<Double> results = cursors
-        .map(cursor -> {
-          final BufferAggregator bufferAggregator = aggregatorFactory.apply(cursor.getColumnSelectorFactory());
-          bufferAggregator.init(aggregationBuffer, 0);
+      final BufferAggregator bufferAggregator = aggregatorFactory.apply(cursor.getColumnSelectorFactory());
+      bufferAggregator.init(aggregationBuffer, 0);
 
-          while (!cursor.isDone()) {
-            bufferAggregator.aggregate(aggregationBuffer, 0);
-            cursor.advance();
-          }
+      while (!cursor.isDone()) {
+        bufferAggregator.aggregate(aggregationBuffer, 0);
+        cursor.advance();
+      }
 
-          final Double dbl = (Double) bufferAggregator.get(aggregationBuffer, 0);
-          bufferAggregator.close();
-          return dbl;
-        })
-        .toList();
+      final Double dbl = (Double) bufferAggregator.get(aggregationBuffer, 0);
+      bufferAggregator.close();
 
-    return Iterables.getOnlyElement(results);
+      return dbl;
+    }
   }
 
   private static class NativeBufferAggregator implements BufferAggregator
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionFilterBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionFilterBenchmark.java
index e7d8ad00bc5..06f0952b51f 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionFilterBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionFilterBenchmark.java
@@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableList;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.ExpressionProcessing;
 import org.apache.druid.query.expression.TestExprMacroTable;
@@ -33,9 +32,10 @@ import org.apache.druid.query.filter.ExpressionDimFilter;
 import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.generator.GeneratorColumnSchema;
 import org.apache.druid.segment.generator.GeneratorSchemaInfo;
@@ -58,7 +58,6 @@ import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.util.Arrays;
-import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 @State(Scope.Benchmark)
@@ -146,52 +145,34 @@ public class ExpressionFilterBenchmark
   @Benchmark
   public void expressionFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        expressionFilter.toFilter(),
-        index.getDataInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("x");
-          consumeString(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(expressionFilter.toFilter())
+                                                     .build();
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
 
-    blackhole.consume(results);
+
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("x");
+      while (!cursor.isDone()) {
+        blackhole.consume(selector.getObject());
+        cursor.advance();
+      }
+    }
   }
 
   @Benchmark
   public void nativeFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        nativeFilter.toFilter(),
-        index.getDataInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("x");
-          consumeString(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
-  }
-
-  private void consumeString(final Cursor cursor, final ColumnValueSelector selector, final Blackhole blackhole)
-  {
-    while (!cursor.isDone()) {
-      blackhole.consume(selector.getLong());
-      cursor.advance();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(nativeFilter.toFilter())
+                                                     .build();
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("x");
+      while (!cursor.isDone()) {
+        blackhole.consume(selector.getObject());
+        cursor.advance();
+      }
     }
   }
 }
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java
index dd33b660644..5bc67bc9f72 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java
@@ -25,8 +25,10 @@ import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.ExpressionProcessing;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.ExtractionDimensionSpec;
 import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable;
@@ -35,9 +37,12 @@ import org.apache.druid.query.extraction.StrlenExtractionFn;
 import org.apache.druid.query.extraction.TimeFormatExtractionFn;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
@@ -62,8 +67,8 @@ import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
+
 import java.util.BitSet;
-import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 @State(Scope.Benchmark)
@@ -141,388 +146,281 @@ public class ExpressionSelectorBenchmark
   @Benchmark
   public void timeFloorUsingExpression(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "timestamp_floor(__time, 'PT1H')",
-                    ColumnType.LONG,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "timestamp_floor(__time, 'PT1H')",
+                                                                 ColumnType.LONG,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void timeFloorUsingExtractionFn(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final DimensionSelector selector = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(
-                  new ExtractionDimensionSpec(
-                      ColumnHolder.TIME_COLUMN_NAME,
-                      "v",
-                      new TimeFormatExtractionFn(null, null, null, Granularities.HOUR, true)
-                  )
-              );
-          consumeDimension(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+      final DimensionSelector selector = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(
+              new ExtractionDimensionSpec(
+                  ColumnHolder.TIME_COLUMN_NAME,
+                  "v",
+                  new TimeFormatExtractionFn(null, null, null, Granularities.HOUR, true)
+              )
+          );
+      consumeDimension(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void timeFloorUsingCursor(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.HOUR,
-        false,
-        null
-    );
+    final StorageAdapter adapter = new QueryableIndexStorageAdapter(index);
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final CursorGranularizer granularizer = CursorGranularizer.create(
+          adapter,
+          cursor,
+          Granularities.HOUR,
+          adapter.getInterval(),
+          false
+      );
+      final Sequence<Long> results =
+          Sequences.simple(granularizer.getBucketIterable())
+                   .map(bucketInterval -> {
+                     if (!granularizer.advanceToBucket(bucketInterval)) {
+                       return 0L;
+                     }
+                     long count = 0L;
+                     while (!cursor.isDone()) {
+                       count++;
+                       if (!granularizer.advanceCursorWithinBucket()) {
+                         break;
+                       }
+                     }
+                     return count;
+                   });
 
-    final List<Long> results = cursors
-        .map(cursor -> {
-          long count = 0L;
-          while (!cursor.isDone()) {
-            count++;
-            cursor.advance();
-          }
-          return count;
-        })
-        .toList();
+      long count = 0L;
+      for (Long result : results.toList()) {
+        count += result;
+      }
 
-    long count = 0L;
-    for (Long result : results) {
-      count += result;
+      blackhole.consume(count);
     }
-
-    blackhole.consume(count);
   }
 
   @Benchmark
   public void timeFormatUsingExpression(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "timestamp_format(__time, 'yyyy-MM-dd')",
-                    ColumnType.STRING,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "timestamp_format(__time, 'yyyy-MM-dd')",
+                                                                 ColumnType.STRING,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final DimensionSelector selector = cursor.getColumnSelectorFactory().makeDimensionSelector(
-              DefaultDimensionSpec.of("v")
-          );
-          consumeDimension(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final DimensionSelector selector = cursor.getColumnSelectorFactory().makeDimensionSelector(
+          DefaultDimensionSpec.of("v")
+      );
+      consumeDimension(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void timeFormatUsingExtractionFn(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    final List<?> results = cursors
-        .map(cursor -> {
-          final DimensionSelector selector = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(
-                  new ExtractionDimensionSpec(
-                      ColumnHolder.TIME_COLUMN_NAME,
-                      "v",
-                      new TimeFormatExtractionFn("yyyy-MM-dd", null, null, null, false)
-                  )
-              );
-          consumeDimension(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final DimensionSelector selector = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(
+              new ExtractionDimensionSpec(
+                  ColumnHolder.TIME_COLUMN_NAME,
+                  "v",
+                  new TimeFormatExtractionFn("yyyy-MM-dd", null, null, null, false)
+              )
+          );
+      consumeDimension(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void strlenUsingExpressionAsLong(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "strlen(s)",
-                    ColumnType.STRING,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "strlen(s)",
+                                                                 ColumnType.STRING,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void strlenUsingExpressionAsString(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "strlen(s)",
-                    ColumnType.STRING,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "strlen(s)",
+                                                                 ColumnType.STRING,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final DimensionSelector selector = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec("v", "v", ColumnType.STRING));
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final DimensionSelector selector = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec("v", "v", ColumnType.STRING));
 
-          consumeDimension(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+      consumeDimension(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void strlenUsingExtractionFn(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final DimensionSelector selector = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new ExtractionDimensionSpec("x", "v", StrlenExtractionFn.instance()));
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final DimensionSelector selector = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new ExtractionDimensionSpec("x", "v", StrlenExtractionFn.instance()));
-
-          consumeDimension(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+      consumeDimension(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void arithmeticOnLong(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "n + 1",
-                    ColumnType.LONG,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "n + 1",
+                                                                 ColumnType.LONG,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void stringConcatAndCompareOnLong(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "concat(n, ' is my favorite number') == '3 is my favorite number'",
-                    ColumnType.LONG,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "concat(n, ' is my favorite number') == '3 is my favorite number'",
+                                                                 ColumnType.LONG,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void caseSearched1(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "case_searched(s == 'asd' || isnull(s) || s == 'xxx', 1, s == 'foo' || s == 'bar', 2, 3)",
-                    ColumnType.LONG,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "case_searched(s == 'asd' || isnull(s) || s == 'xxx', 1, s == 'foo' || s == 'bar', 2, 3)",
+                                                                 ColumnType.LONG,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void caseSearched2(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "case_searched(s == 'asd' || isnull(s) || n == 1, 1, n == 2, 2, 3)",
-                    ColumnType.LONG,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "case_searched(s == 'asd' || isnull(s) || n == 1, 1, n == 2, 2, 3)",
+                                                                 ColumnType.LONG,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
 
   @Benchmark
   public void caseSearched100(Blackhole blackhole)
   {
-
     StringBuilder caseBranches = new StringBuilder();
     for (int i = 0; i < 100; i++) {
       caseBranches.append(
@@ -534,115 +432,90 @@ public class ExpressionSelectorBenchmark
       );
     }
 
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "case_searched(s == 'asd' || isnull(s) || n == 1, 1, " + caseBranches + " 3)",
-                    ColumnType.LONG,
-                    TestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "case_searched(s == 'asd' || isnull(s) || n == 1, 1, " + caseBranches + " 3)",
+                                                                 ColumnType.LONG,
+                                                                 TestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void caseSearchedWithLookup(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "v",
-                    "case_searched(n == 1001, -1, "
-                        + "lookup(s, 'lookyloo') == 'asd1', 1, "
-                        + "lookup(s, 'lookyloo') == 'asd2', 2, "
-                        + "lookup(s, 'lookyloo') == 'asd3', 3, "
-                        + "lookup(s, 'lookyloo') == 'asd4', 4, "
-                        + "lookup(s, 'lookyloo') == 'asd5', 5, "
-                        + "-2)",
-                    ColumnType.LONG,
-                    LookupEnabledTestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             new ExpressionVirtualColumn(
+                                                                 "v",
+                                                                 "case_searched(n == 1001, -1, "
+                                                                 + "lookup(s, 'lookyloo') == 'asd1', 1, "
+                                                                 + "lookup(s, 'lookyloo') == 'asd2', 2, "
+                                                                 + "lookup(s, 'lookyloo') == 'asd3', 3, "
+                                                                 + "lookup(s, 'lookyloo') == 'asd4', 4, "
+                                                                 + "lookup(s, 'lookyloo') == 'asd5', 5, "
+                                                                 + "-2)",
+                                                                 ColumnType.LONG,
+                                                                 LookupEnabledTestExprMacroTable.INSTANCE
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
   @Benchmark
   public void caseSearchedWithLookup2(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "ll",
-                    "lookup(s, 'lookyloo')",
-                    ColumnType.STRING,
-                    LookupEnabledTestExprMacroTable.INSTANCE
-                ),
-                new ExpressionVirtualColumn(
-                    "v",
-                    "case_searched(n == 1001, -1, "
-                        + "ll == 'asd1', 1, "
-                        + "ll == 'asd2', 2, "
-                        + "ll == 'asd3', 3, "
-                        + "ll == 'asd4', 4, "
-                        + "ll == 'asd5', 5, "
-                        + "-2)",
-                    ColumnType.LONG,
-                    LookupEnabledTestExprMacroTable.INSTANCE
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             ImmutableList.of(
+                                                                 new ExpressionVirtualColumn(
+                                                                     "ll",
+                                                                     "lookup(s, 'lookyloo')",
+                                                                     ColumnType.STRING,
+                                                                     LookupEnabledTestExprMacroTable.INSTANCE
+                                                                 ),
+                                                                 new ExpressionVirtualColumn(
+                                                                     "v",
+                                                                     "case_searched(n == 1001, -1, "
+                                                                     + "ll == 'asd1', 1, "
+                                                                     + "ll == 'asd2', 2, "
+                                                                     + "ll == 'asd3', 3, "
+                                                                     + "ll == 'asd4', 4, "
+                                                                     + "ll == 'asd5', 5, "
+                                                                     + "-2)",
+                                                                     ColumnType.LONG,
+                                                                     LookupEnabledTestExprMacroTable.INSTANCE
+                                                                 )
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
 
-    final List<?> results = cursors
-        .map(cursor -> {
-          final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-          consumeLong(cursor, selector, blackhole);
-          return null;
-        })
-        .toList();
-
-    blackhole.consume(results);
+    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      consumeLong(cursor, selector, blackhole);
+    }
   }
 
 
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
index 754eb7b6d38..5604f7fe883 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
@@ -22,7 +22,6 @@ package org.apache.druid.benchmark;
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
@@ -33,6 +32,8 @@ import org.apache.druid.query.expression.TestExprMacroTable;
 import org.apache.druid.segment.ColumnCache;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
@@ -150,15 +151,14 @@ public class ExpressionVectorSelectorBenchmark
             )
         )
     );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(virtualColumns)
+                                                     .build();
+    final CursorHolder cursorHolder = closer.register(
+        new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)
+    );
     if (vectorize) {
-      VectorCursor cursor = new QueryableIndexStorageAdapter(index).makeVectorCursor(
-          null,
-          index.getDataInterval(),
-          virtualColumns,
-          false,
-          512,
-          null
-      );
+      VectorCursor cursor = cursorHolder.asVectorCursor();
       if (outputType.isNumeric()) {
         VectorValueSelector selector = cursor.getColumnSelectorFactory().makeValueSelector("v");
         if (outputType.is(ExprType.DOUBLE)) {
@@ -174,29 +174,34 @@ public class ExpressionVectorSelectorBenchmark
             cursor.advance();
           }
         }
-        closer.register(cursor);
       }
     } else {
-      Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-          null,
-          index.getDataInterval(),
-          virtualColumns,
-          Granularities.ALL,
-          false,
-          null
-      );
-
-      int rowCount = cursors
-          .map(cursor -> {
-            final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
-            int rows = 0;
-            while (!cursor.isDone()) {
-              blackhole.consume(selector.getObject());
-              rows++;
-              cursor.advance();
-            }
-            return rows;
-          }).accumulate(0, (acc, in) -> acc + in);
+      final Cursor cursor = cursorHolder.asCursor();
+      final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      int rowCount = 0;
+      if (outputType.isNumeric()) {
+        if (outputType.is(ExprType.DOUBLE)) {
+          while (!cursor.isDone()) {
+            blackhole.consume(selector.isNull());
+            blackhole.consume(selector.getDouble());
+            rowCount++;
+            cursor.advance();
+          }
+        } else {
+          while (!cursor.isDone()) {
+            blackhole.consume(selector.isNull());
+            blackhole.consume(selector.getLong());
+            rowCount++;
+            cursor.advance();
+          }
+        }
+      } else {
+        while (!cursor.isDone()) {
+          blackhole.consume(selector.getObject());
+          rowCount++;
+          cursor.advance();
+        }
+      }
 
       blackhole.consume(rowCount);
     }
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
index 98c7ddf4f85..65dde3cb65a 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
@@ -25,9 +25,6 @@ import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.FileUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.js.JavaScriptConfig;
 import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde;
@@ -48,6 +45,8 @@ import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.query.ordering.StringComparators;
 import org.apache.druid.segment.BaseLongColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.IndexMergerV9;
@@ -55,7 +54,6 @@ import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnConfig;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.data.IndexedInts;
@@ -234,8 +232,10 @@ public class FilterPartitionBenchmark
   public void stringRead(Blackhole blackhole)
   {
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, null);
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, null)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -244,9 +244,10 @@ public class FilterPartitionBenchmark
   public void longRead(Blackhole blackhole)
   {
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, null);
-
-    readCursorsLong(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, null)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursorLong(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -255,9 +256,10 @@ public class FilterPartitionBenchmark
   public void timeFilterNone(Blackhole blackhole)
   {
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, timeFilterNone);
-
-    readCursorsLong(cursors, blackhole);
+    try (CursorHolder cursorHolder = makeCursorHolder(sa, timeFilterNone)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursorLong(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -266,9 +268,10 @@ public class FilterPartitionBenchmark
   public void timeFilterHalf(Blackhole blackhole)
   {
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, timeFilterHalf);
-
-    readCursorsLong(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, timeFilterHalf)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursorLong(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -277,9 +280,10 @@ public class FilterPartitionBenchmark
   public void timeFilterAll(Blackhole blackhole)
   {
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, timeFilterAll);
-
-    readCursorsLong(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, timeFilterAll)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursorLong(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -290,8 +294,10 @@ public class FilterPartitionBenchmark
     Filter filter = new SelectorFilter("dimSequential", "199");
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, filter);
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, filter)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -302,8 +308,10 @@ public class FilterPartitionBenchmark
     Filter filter = new NoBitmapSelectorFilter("dimSequential", "199");
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, filter);
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, filter)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -314,8 +322,10 @@ public class FilterPartitionBenchmark
     Filter filter = new SelectorDimFilter("dimSequential", "super-199", JS_EXTRACTION_FN).toFilter();
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, filter);
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, filter)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -326,8 +336,10 @@ public class FilterPartitionBenchmark
     Filter filter = new NoBitmapSelectorDimFilter("dimSequential", "super-199", JS_EXTRACTION_FN).toFilter();
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, filter);
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, filter)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -343,8 +355,10 @@ public class FilterPartitionBenchmark
     );
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, andFilter);
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, andFilter)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -357,8 +371,10 @@ public class FilterPartitionBenchmark
     Filter orFilter = new OrFilter(Arrays.asList(filter, filter2));
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, orFilter);
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, orFilter)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -371,8 +387,10 @@ public class FilterPartitionBenchmark
     Filter orFilter = new OrFilter(Arrays.asList(filter, filter2));
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, Filters.toCnf(orFilter));
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, Filters.toCnf(orFilter))) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -408,8 +426,10 @@ public class FilterPartitionBenchmark
     );
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, dimFilter3.toFilter());
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, dimFilter3.toFilter())) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
   @Benchmark
@@ -445,55 +465,46 @@ public class FilterPartitionBenchmark
     );
 
     StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, Filters.toCnf(dimFilter3.toFilter()));
-    readCursors(cursors, blackhole);
+    try (final CursorHolder cursorHolder = makeCursorHolder(sa, Filters.toCnf(dimFilter3.toFilter()))) {
+      final Cursor cursor = cursorHolder.asCursor();
+      readCursor(cursor, blackhole);
+    }
   }
 
-  private Sequence<Cursor> makeCursors(StorageAdapter sa, Filter filter)
+  private CursorHolder makeCursorHolder(StorageAdapter sa, Filter filter)
   {
-    return sa.makeCursors(filter, schemaInfo.getDataInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null);
-  }
-
-  private void readCursors(Sequence<Cursor> cursors, Blackhole blackhole)
-  {
-    final Sequence<Void> voids = Sequences.map(
-        cursors,
-        input -> {
-          List<DimensionSelector> selectors = new ArrayList<>();
-          selectors.add(
-              input.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("dimSequential", null))
-          );
-          while (!input.isDone()) {
-            for (DimensionSelector selector : selectors) {
-              IndexedInts row = selector.getRow();
-              blackhole.consume(selector.lookupName(row.get(0)));
-            }
-            input.advance();
-          }
-          return null;
-        }
+    return sa.makeCursorHolder(
+        CursorBuildSpec.builder()
+                       .setFilter(filter)
+                       .setInterval(schemaInfo.getDataInterval())
+                       .build()
     );
-
-    blackhole.consume(voids.toList());
   }
 
-  private void readCursorsLong(Sequence<Cursor> cursors, final Blackhole blackhole)
+  private void readCursor(Cursor cursor, Blackhole blackhole)
   {
-    final Sequence<Void> voids = Sequences.map(
-        cursors,
-        input -> {
-          BaseLongColumnValueSelector selector = input.getColumnSelectorFactory()
-                                                      .makeColumnValueSelector("sumLongSequential");
-          while (!input.isDone()) {
-            long rowval = selector.getLong();
-            blackhole.consume(rowval);
-            input.advance();
-          }
-          return null;
-        }
+    List<DimensionSelector> selectors = new ArrayList<>();
+    selectors.add(
+        cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("dimSequential", null))
     );
+    while (!cursor.isDone()) {
+      for (DimensionSelector selector : selectors) {
+        IndexedInts row = selector.getRow();
+        blackhole.consume(selector.lookupName(row.get(0)));
+      }
+      cursor.advance();
+    }
+  }
 
-    blackhole.consume(voids.toList());
+  private void readCursorLong(Cursor cursor, final Blackhole blackhole)
+  {
+    BaseLongColumnValueSelector selector = cursor.getColumnSelectorFactory()
+                                                 .makeColumnValueSelector("sumLongSequential");
+    while (!cursor.isDone()) {
+      long rowval = selector.getLong();
+      blackhole.consume(rowval);
+      cursor.advance();
+    }
   }
 
   private static class NoBitmapSelectorFilter extends SelectorFilter
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/IndexedTableJoinCursorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/IndexedTableJoinCursorBenchmark.java
index c2dd8ae9996..952e9b188fe 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/IndexedTableJoinCursorBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/IndexedTableJoinCursorBenchmark.java
@@ -27,7 +27,6 @@ import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.QueryContexts;
@@ -37,6 +36,8 @@ import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexSegment;
@@ -216,9 +217,11 @@ public class IndexedTableJoinCursorBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void hashJoinCursorColumnValueSelectors(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = makeCursors();
-    int rowCount = processRowsValueSelector(blackhole, cursors, projectionColumns);
-    blackhole.consume(rowCount);
+    try (final CursorHolder cursorHolder = makeCursorHolder()) {
+      final Cursor cursor = cursorHolder.asCursor();
+      int rowCount = processRowsValueSelector(blackhole, cursor, projectionColumns);
+      blackhole.consume(rowCount);
+    }
   }
 
   @Benchmark
@@ -226,21 +229,16 @@ public class IndexedTableJoinCursorBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void hashJoinCursorDimensionSelectors(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = makeCursors();
-    int rowCount = processRowsDimensionSelectors(blackhole, cursors, projectionColumns);
-    blackhole.consume(rowCount);
+    try (final CursorHolder cursorHolder = makeCursorHolder()) {
+      final Cursor cursor = cursorHolder.asCursor();
+      int rowCount = processRowsDimensionSelectors(blackhole, cursor, projectionColumns);
+      blackhole.consume(rowCount);
+    }
   }
 
-  private Sequence<Cursor> makeCursors()
+  private CursorHolder makeCursorHolder()
   {
-    return hashJoinSegment.asStorageAdapter().makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    return hashJoinSegment.asStorageAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN);
   }
 
 
@@ -313,106 +311,96 @@ public class IndexedTableJoinCursorBenchmark
 
   private static int processRowsDimensionSelectors(
       final Blackhole blackhole,
-      final Sequence<Cursor> cursors,
+      final Cursor cursor,
       final Set<String> columns
   )
   {
     if (columns.size() == 1) {
-      return processRowsSingleDimensionSelector(blackhole, cursors, Iterables.getOnlyElement(columns));
+      return processRowsSingleDimensionSelector(blackhole, cursor, Iterables.getOnlyElement(columns));
     }
-    return cursors.map(
-        cursor -> {
-          List<DimensionSelector> selectors = columns.stream().map(column -> {
-            ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
-            return factory.makeDimensionSelector(DefaultDimensionSpec.of(column));
-          }).collect(Collectors.toList());
+    List<DimensionSelector> selectors = columns.stream().map(column -> {
+      ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
+      return factory.makeDimensionSelector(DefaultDimensionSpec.of(column));
+    }).collect(Collectors.toList());
 
-          int rowCount = 0;
-          while (!cursor.isDone()) {
-            for (DimensionSelector selector : selectors) {
-              if (selector.getValueCardinality() < 0) {
-                final IndexedInts row = selector.getRow();
-                final int sz = row.size();
-                for (int i = 0; i < sz; i++) {
-                  blackhole.consume(selector.lookupName(row.get(i)));
-                }
-              } else {
-                final IndexedInts row = selector.getRow();
-                final int sz = row.size();
-                for (int i = 0; i < sz; i++) {
-                  blackhole.consume(row.get(i));
-                }
-              }
-            }
-
-            rowCount++;
-            cursor.advance();
+    int rowCount = 0;
+    while (!cursor.isDone()) {
+      for (DimensionSelector selector : selectors) {
+        if (selector.getValueCardinality() < 0) {
+          final IndexedInts row = selector.getRow();
+          final int sz = row.size();
+          for (int i = 0; i < sz; i++) {
+            blackhole.consume(selector.lookupName(row.get(i)));
           }
-          return rowCount;
-        }).accumulate(0, (acc, in) -> acc + in);
+        } else {
+          final IndexedInts row = selector.getRow();
+          final int sz = row.size();
+          for (int i = 0; i < sz; i++) {
+            blackhole.consume(row.get(i));
+          }
+        }
+      }
+
+      rowCount++;
+      cursor.advance();
+    }
+    return rowCount;
   }
 
   private static int processRowsSingleDimensionSelector(
       final Blackhole blackhole,
-      final Sequence<Cursor> cursors,
+      final Cursor cursor,
       final String dimension
   )
   {
-    return cursors.map(
-        cursor -> {
-          final DimensionSelector selector = cursor.getColumnSelectorFactory()
-                                                   .makeDimensionSelector(DefaultDimensionSpec.of(dimension));
+    final DimensionSelector selector = cursor.getColumnSelectorFactory()
+                                             .makeDimensionSelector(DefaultDimensionSpec.of(dimension));
 
-          int rowCount = 0;
-          if (selector.getValueCardinality() < 0) {
-            String lastValue;
-            while (!cursor.isDone()) {
-              final IndexedInts row = selector.getRow();
-              final int sz = row.size();
-              for (int i = 0; i < sz; i++) {
-                lastValue = selector.lookupName(row.get(i));
-                blackhole.consume(lastValue);
-              }
-              rowCount++;
-              cursor.advance();
-            }
-            return rowCount;
-          } else {
-            int lastValue;
-            while (!cursor.isDone()) {
-              final IndexedInts row = selector.getRow();
-              final int sz = row.size();
-              for (int i = 0; i < sz; i++) {
-                lastValue = row.get(i);
-                blackhole.consume(lastValue);
-              }
-              rowCount++;
-              cursor.advance();
-            }
-            return rowCount;
-          }
+    int rowCount = 0;
+    if (selector.getValueCardinality() < 0) {
+      String lastValue;
+      while (!cursor.isDone()) {
+        final IndexedInts row = selector.getRow();
+        final int sz = row.size();
+        for (int i = 0; i < sz; i++) {
+          lastValue = selector.lookupName(row.get(i));
+          blackhole.consume(lastValue);
         }
-    ).accumulate(0, (acc, in) -> acc + in);
+        rowCount++;
+        cursor.advance();
+      }
+      return rowCount;
+    } else {
+      int lastValue;
+      while (!cursor.isDone()) {
+        final IndexedInts row = selector.getRow();
+        final int sz = row.size();
+        for (int i = 0; i < sz; i++) {
+          lastValue = row.get(i);
+          blackhole.consume(lastValue);
+        }
+        rowCount++;
+        cursor.advance();
+      }
+      return rowCount;
+    }
   }
 
-  private static int processRowsValueSelector(final Blackhole blackhole, final Sequence<Cursor> cursors, final Set<String> columns)
+  private static int processRowsValueSelector(final Blackhole blackhole, final Cursor cursor, final Set<String> columns)
   {
-    return cursors.map(
-        cursor -> {
-          ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
+    ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
 
-          List<BaseObjectColumnValueSelector> selectors =
-              columns.stream().map(factory::makeColumnValueSelector).collect(Collectors.toList());
-          int rowCount = 0;
-          while (!cursor.isDone()) {
-            for (BaseObjectColumnValueSelector<?> selector : selectors) {
-              blackhole.consume(selector.getObject());
-            }
+    List<BaseObjectColumnValueSelector> selectors =
+        columns.stream().map(factory::makeColumnValueSelector).collect(Collectors.toList());
+    int rowCount = 0;
+    while (!cursor.isDone()) {
+      for (BaseObjectColumnValueSelector<?> selector : selectors) {
+        blackhole.consume(selector.getObject());
+      }
 
-            rowCount++;
-            cursor.advance();
-          }
-          return rowCount;
-        }).accumulate(0, (acc, in) -> acc + in);
+      rowCount++;
+      cursor.advance();
+    }
+    return rowCount;
   }
 }
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/JoinAndLookupBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/JoinAndLookupBenchmark.java
index ba07091fe5a..c66ef6baf0a 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/JoinAndLookupBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/JoinAndLookupBenchmark.java
@@ -23,18 +23,18 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.FileUtils;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.expression.LookupExprMacro;
+import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.query.lookup.LookupExtractorFactoryContainer;
 import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider;
 import org.apache.druid.query.lookup.MapLookupExtractorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexSegment;
@@ -334,38 +334,34 @@ public class JoinAndLookupBenchmark
     );
   }
 
-  private static String getLastValue(final Sequence<Cursor> cursors, final String dimension)
+  private static String getLastValue(final Cursor cursor, final String dimension)
   {
-    return cursors.map(
-        cursor -> {
-          final DimensionSelector selector = cursor.getColumnSelectorFactory()
-                                                   .makeDimensionSelector(DefaultDimensionSpec.of(dimension));
+    final DimensionSelector selector = cursor.getColumnSelectorFactory()
+                                             .makeDimensionSelector(DefaultDimensionSpec.of(dimension));
 
-          if (selector.getValueCardinality() < 0) {
-            String lastValue = null;
-            while (!cursor.isDone()) {
-              final IndexedInts row = selector.getRow();
-              final int sz = row.size();
-              for (int i = 0; i < sz; i++) {
-                lastValue = selector.lookupName(row.get(i));
-              }
-              cursor.advance();
-            }
-            return lastValue;
-          } else {
-            int lastValue = -1;
-            while (!cursor.isDone()) {
-              final IndexedInts row = selector.getRow();
-              final int sz = row.size();
-              for (int i = 0; i < sz; i++) {
-                lastValue = row.get(i);
-              }
-              cursor.advance();
-            }
-            return selector.lookupName(lastValue);
-          }
+    if (selector.getValueCardinality() < 0) {
+      String lastValue = null;
+      while (!cursor.isDone()) {
+        final IndexedInts row = selector.getRow();
+        final int sz = row.size();
+        for (int i = 0; i < sz; i++) {
+          lastValue = selector.lookupName(row.get(i));
         }
-    ).accumulate(null, (acc, in) -> in);
+        cursor.advance();
+      }
+      return lastValue;
+    } else {
+      int lastValue = -1;
+      while (!cursor.isDone()) {
+        final IndexedInts row = selector.getRow();
+        final int sz = row.size();
+        for (int i = 0; i < sz; i++) {
+          lastValue = row.get(i);
+        }
+        cursor.advance();
+      }
+      return selector.lookupName(lastValue);
+    }
   }
 
   @Benchmark
@@ -373,16 +369,10 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void baseSegment(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = baseSegment.asStorageAdapter().makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "countryIsoCode"));
+    try (final CursorHolder cursorHolder = baseSegment.asStorageAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "countryIsoCode"));
+    }
   }
 
   @Benchmark
@@ -390,16 +380,14 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void baseSegmentWithFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = baseSegment.asStorageAdapter().makeCursors(
-        new SelectorDimFilter("countryIsoCode", "CA", null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "countryIsoCode"));
+    final Filter filter = new SelectorDimFilter("countryIsoCode", "CA", null).toFilter();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(filter)
+                                                     .build();
+    try (final CursorHolder cursorHolder = baseSegment.asStorageAdapter().makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "countryIsoCode"));
+    }
   }
 
   @Benchmark
@@ -407,16 +395,11 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void joinLookupStringKey(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = hashJoinLookupStringKeySegment.asStorageAdapter().makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "c.v"));
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "c.v"));
+    }
   }
 
   @Benchmark
@@ -424,16 +407,15 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void joinLookupStringKeyWithFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = hashJoinLookupStringKeySegment.asStorageAdapter().makeCursors(
-        new SelectorDimFilter("c.v", "Canada", null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "c.v"));
+    final Filter filter = new SelectorDimFilter("c.v", "Canada", null).toFilter();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(filter)
+                                                     .build();
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "c.v"));
+    }
   }
 
   @Benchmark
@@ -441,16 +423,11 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void joinLookupLongKey(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = hashJoinLookupLongKeySegment.asStorageAdapter().makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "c.v"));
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "c.v"));
+    }
   }
 
   @Benchmark
@@ -458,16 +435,15 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void joinLookupLongKeyWithFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = hashJoinLookupLongKeySegment.asStorageAdapter().makeCursors(
-        new SelectorDimFilter("c.v", "Canada", null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "c.v"));
+    final Filter filter = new SelectorDimFilter("c.v", "Canada", null).toFilter();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(filter)
+                                                     .build();
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "c.v"));
+    }
   }
 
   @Benchmark
@@ -475,16 +451,11 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void joinIndexedTableLongKey(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = hashJoinIndexedTableLongKeySegment.asStorageAdapter().makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "c.countryName"));
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "c.countryName"));
+    }
   }
 
   @Benchmark
@@ -492,16 +463,15 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void joinIndexedTableLongKeyWithFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = hashJoinIndexedTableLongKeySegment.asStorageAdapter().makeCursors(
-        new SelectorDimFilter("c.countryName", "Canada", null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "c.countryName"));
+    final Filter filter = new SelectorDimFilter("c.countryName", "Canada", null).toFilter();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(filter)
+                                                     .build();
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "c.countryName"));
+    }
   }
 
   @Benchmark
@@ -509,16 +479,11 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void joinIndexedTableStringKey(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = hashJoinIndexedTableStringKeySegment.asStorageAdapter().makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "c.countryName"));
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "c.countryName"));
+    }
   }
 
   @Benchmark
@@ -526,16 +491,15 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void joinIndexedTableStringKeyWithFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = hashJoinIndexedTableStringKeySegment.asStorageAdapter().makeCursors(
-        new SelectorDimFilter("c.countryName", "Canada", null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, "c.countryName"));
+    final Filter filter = new SelectorDimFilter("c.countryName", "Canada", null).toFilter();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(filter)
+                                                     .build();
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, "c.countryName"));
+    }
   }
 
   @Benchmark
@@ -543,16 +507,14 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void lookupVirtualColumnStringKey(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = baseSegment.asStorageAdapter().makeCursors(
-        null,
-        Intervals.ETERNITY,
-        lookupVirtualColumns,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, LOOKUP_COUNTRY_CODE_TO_NAME));
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(lookupVirtualColumns)
+                                                     .build();
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, LOOKUP_COUNTRY_CODE_TO_NAME));
+    }
   }
 
   @Benchmark
@@ -560,16 +522,16 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void lookupVirtualColumnStringKeyWithFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = baseSegment.asStorageAdapter().makeCursors(
-        new SelectorDimFilter(LOOKUP_COUNTRY_CODE_TO_NAME, "Canada", null).toFilter(),
-        Intervals.ETERNITY,
-        lookupVirtualColumns,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, LOOKUP_COUNTRY_CODE_TO_NAME));
+    final Filter filter = new SelectorDimFilter(LOOKUP_COUNTRY_CODE_TO_NAME, "Canada", null).toFilter();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(filter)
+                                                     .setVirtualColumns(lookupVirtualColumns)
+                                                     .build();
+    try (final CursorHolder cursorHolder = hashJoinLookupStringKeySegment.asStorageAdapter()
+                                                                         .makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, LOOKUP_COUNTRY_CODE_TO_NAME));
+    }
   }
 
   @Benchmark
@@ -577,16 +539,13 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void lookupVirtualColumnLongKey(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = baseSegment.asStorageAdapter().makeCursors(
-        null,
-        Intervals.ETERNITY,
-        lookupVirtualColumns,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, LOOKUP_COUNTRY_NUMBER_TO_NAME));
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(lookupVirtualColumns)
+                                                     .build();
+    try (final CursorHolder cursorHolder = baseSegment.asStorageAdapter().makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, LOOKUP_COUNTRY_NUMBER_TO_NAME));
+    }
   }
 
   @Benchmark
@@ -594,15 +553,14 @@ public class JoinAndLookupBenchmark
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   public void lookupVirtualColumnLongKeyWithFilter(Blackhole blackhole)
   {
-    final Sequence<Cursor> cursors = baseSegment.asStorageAdapter().makeCursors(
-        new SelectorDimFilter(LOOKUP_COUNTRY_NUMBER_TO_NAME, "Canada", null).toFilter(),
-        Intervals.ETERNITY,
-        lookupVirtualColumns,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    blackhole.consume(getLastValue(cursors, LOOKUP_COUNTRY_NUMBER_TO_NAME));
+    final Filter filter = new SelectorDimFilter(LOOKUP_COUNTRY_NUMBER_TO_NAME, "Canada", null).toFilter();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(lookupVirtualColumns)
+                                                     .setFilter(filter)
+                                                     .build();
+    try (final CursorHolder cursorHolder = baseSegment.asStorageAdapter().makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      blackhole.consume(getLastValue(cursor, LOOKUP_COUNTRY_NUMBER_TO_NAME));
+    }
   }
 }
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java
index 37988598324..6f0796f412f 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java
@@ -20,8 +20,6 @@
 package org.apache.druid.benchmark.indexing;
 
 import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.js.JavaScriptConfig;
 import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde;
@@ -36,8 +34,9 @@ import org.apache.druid.query.filter.SearchQueryDimFilter;
 import org.apache.druid.query.ordering.StringComparators;
 import org.apache.druid.query.search.ContainsSearchQuerySpec;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.data.IndexedInts;
 import org.apache.druid.segment.generator.DataGenerator;
 import org.apache.druid.segment.generator.GeneratorBasicSchemas;
@@ -149,22 +148,23 @@ public class IncrementalIndexReadBenchmark
   public void read(Blackhole blackhole)
   {
     IncrementalIndexStorageAdapter sa = new IncrementalIndexStorageAdapter(incIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, null);
-    Cursor cursor = cursors.limit(1).toList().get(0);
+    try (final CursorHolder cursorHolder = makeCursor(sa, null)) {
+      Cursor cursor = cursorHolder.asCursor();
 
-    List<DimensionSelector> selectors = new ArrayList<>();
-    selectors.add(makeDimensionSelector(cursor, "dimSequential"));
-    selectors.add(makeDimensionSelector(cursor, "dimZipf"));
-    selectors.add(makeDimensionSelector(cursor, "dimUniform"));
-    selectors.add(makeDimensionSelector(cursor, "dimSequentialHalfNull"));
+      List<DimensionSelector> selectors = new ArrayList<>();
+      selectors.add(makeDimensionSelector(cursor, "dimSequential"));
+      selectors.add(makeDimensionSelector(cursor, "dimZipf"));
+      selectors.add(makeDimensionSelector(cursor, "dimUniform"));
+      selectors.add(makeDimensionSelector(cursor, "dimSequentialHalfNull"));
 
-    cursor.reset();
-    while (!cursor.isDone()) {
-      for (DimensionSelector selector : selectors) {
-        IndexedInts row = selector.getRow();
-        blackhole.consume(selector.lookupName(row.get(0)));
+      cursor.reset();
+      while (!cursor.isDone()) {
+        for (DimensionSelector selector : selectors) {
+          IndexedInts row = selector.getRow();
+          blackhole.consume(selector.lookupName(row.get(0)));
+        }
+        cursor.advance();
       }
-      cursor.advance();
     }
   }
 
@@ -184,35 +184,34 @@ public class IncrementalIndexReadBenchmark
     );
 
     IncrementalIndexStorageAdapter sa = new IncrementalIndexStorageAdapter(incIndex);
-    Sequence<Cursor> cursors = makeCursors(sa, filter);
-    Cursor cursor = cursors.limit(1).toList().get(0);
+    try (final CursorHolder cursorHolder = makeCursor(sa, filter)) {
+      Cursor cursor = cursorHolder.asCursor();
 
-    List<DimensionSelector> selectors = new ArrayList<>();
-    selectors.add(makeDimensionSelector(cursor, "dimSequential"));
-    selectors.add(makeDimensionSelector(cursor, "dimZipf"));
-    selectors.add(makeDimensionSelector(cursor, "dimUniform"));
-    selectors.add(makeDimensionSelector(cursor, "dimSequentialHalfNull"));
+      List<DimensionSelector> selectors = new ArrayList<>();
+      selectors.add(makeDimensionSelector(cursor, "dimSequential"));
+      selectors.add(makeDimensionSelector(cursor, "dimZipf"));
+      selectors.add(makeDimensionSelector(cursor, "dimUniform"));
+      selectors.add(makeDimensionSelector(cursor, "dimSequentialHalfNull"));
 
-    cursor.reset();
-    while (!cursor.isDone()) {
-      for (DimensionSelector selector : selectors) {
-        IndexedInts row = selector.getRow();
-        blackhole.consume(selector.lookupName(row.get(0)));
+      cursor.reset();
+      while (!cursor.isDone()) {
+        for (DimensionSelector selector : selectors) {
+          IndexedInts row = selector.getRow();
+          blackhole.consume(selector.lookupName(row.get(0)));
+        }
+        cursor.advance();
       }
-      cursor.advance();
     }
   }
 
-  private Sequence<Cursor> makeCursors(IncrementalIndexStorageAdapter sa, DimFilter filter)
+  private CursorHolder makeCursor(IncrementalIndexStorageAdapter sa, DimFilter filter)
   {
-    return sa.makeCursors(
-        filter == null ? null : filter.toFilter(),
-        schemaInfo.getDataInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    CursorBuildSpec.CursorBuildSpecBuilder builder = CursorBuildSpec.builder()
+                                                                    .setInterval(schemaInfo.getDataInterval());
+    if (filter != null) {
+      builder.setFilter(filter.toFilter());
+    }
+    return sa.makeCursorHolder(builder.build());
   }
 
   private static DimensionSelector makeDimensionSelector(Cursor cursor, String name)
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java
index 346b8e4a7f7..e7220cc286d 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java
@@ -121,12 +121,12 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 
 @State(Scope.Benchmark)
-@Fork(value = 2)
-@Warmup(iterations = 10)
-@Measurement(iterations = 25)
+@Fork(value = 1)
+@Warmup(iterations = 5)
+@Measurement(iterations = 15)
 public class GroupByBenchmark
 {
-  @Param({"2", "4"})
+  @Param({"4"})
   private int numProcessingThreads;
 
   @Param({"-1"})
@@ -141,7 +141,7 @@ public class GroupByBenchmark
   @Param({"all", "day"})
   private String queryGranularity;
 
-  @Param({"force", "false"})
+  @Param({"false", "force"})
   private String vectorize;
 
   private static final Logger log = new Logger(GroupByBenchmark.class);
@@ -514,7 +514,7 @@ public class GroupByBenchmark
   @State(Scope.Benchmark)
   public static class IncrementalIndexState
   {
-    @Param({"onheap", "offheap"})
+    @Param({"onheap"})
     private String indexType;
 
     IncrementalIndex incIndex;
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java
index a33dbc0dcfc..8143e4e8c75 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java
@@ -34,6 +34,7 @@ import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.DefaultGenericQueryMetricsFactory;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.FinalizeResultsQueryRunner;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryRunner;
@@ -116,7 +117,7 @@ public class ScanBenchmark
   private int limit;
 
   @Param({"NONE", "DESCENDING", "ASCENDING"})
-  private static ScanQuery.Order ordering;
+  private static Order ordering;
 
   private static final Logger log = new Logger(ScanBenchmark.class);
   private static final int RNG_SEED = 9999;
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java
index 97d20ca68ae..5ea82211091 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java
@@ -26,7 +26,7 @@ import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.concurrent.Execs;
-import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.Druids;
@@ -101,8 +101,8 @@ import java.util.concurrent.TimeUnit;
 
 @State(Scope.Benchmark)
 @Fork(value = 1)
-@Warmup(iterations = 10)
-@Measurement(iterations = 25)
+@Warmup(iterations = 5)
+@Measurement(iterations = 15)
 public class TimeseriesBenchmark
 {
   @Param({"750000"})
@@ -114,6 +114,9 @@ public class TimeseriesBenchmark
   @Param({"true", "false"})
   private boolean descending;
 
+  @Param({"all", "hour"})
+  private String queryGranularity;
+
   private static final Logger log = new Logger(TimeseriesBenchmark.class);
   private static final int RNG_SEED = 9999;
   private static final IndexMergerV9 INDEX_MERGER_V9;
@@ -162,7 +165,7 @@ public class TimeseriesBenchmark
       TimeseriesQuery queryA =
           Druids.newTimeseriesQueryBuilder()
                 .dataSource("blah")
-                .granularity(Granularities.ALL)
+                .granularity(Granularity.fromString(queryGranularity))
                 .intervals(intervalSpec)
                 .aggregators(queryAggs)
                 .descending(descending)
@@ -182,7 +185,7 @@ public class TimeseriesBenchmark
       TimeseriesQuery timeFilterQuery =
           Druids.newTimeseriesQueryBuilder()
                 .dataSource("blah")
-                .granularity(Granularities.ALL)
+                .granularity(Granularity.fromString(queryGranularity))
                 .intervals(intervalSpec)
                 .aggregators(queryAggs)
                 .descending(descending)
@@ -202,7 +205,7 @@ public class TimeseriesBenchmark
       TimeseriesQuery timeFilterQuery =
           Druids.newTimeseriesQueryBuilder()
                 .dataSource("blah")
-                .granularity(Granularities.ALL)
+                .granularity(Granularity.fromString(queryGranularity))
                 .intervals(intervalSpec)
                 .aggregators(queryAggs)
                 .descending(descending)
@@ -219,7 +222,7 @@ public class TimeseriesBenchmark
       TimeseriesQuery timeFilterQuery =
           Druids.newTimeseriesQueryBuilder()
                 .dataSource("blah")
-                .granularity(Granularities.ALL)
+                .granularity(Granularity.fromString(queryGranularity))
                 .intervals(intervalSpec)
                 .aggregators(queryAggs)
                 .descending(descending)
@@ -271,7 +274,7 @@ public class TimeseriesBenchmark
   @State(Scope.Benchmark)
   public static class IncrementalIndexState
   {
-    @Param({"onheap", "offheap"})
+    @Param({"onheap"})
     private String indexType;
 
     IncrementalIndex incIndex;
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java
index e69bd15a5f0..655b0d2a423 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java
@@ -26,7 +26,7 @@ import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.concurrent.Execs;
-import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.offheap.OffheapBufferGenerator;
@@ -98,8 +98,8 @@ import java.util.concurrent.TimeUnit;
 
 @State(Scope.Benchmark)
 @Fork(value = 1)
-@Warmup(iterations = 10)
-@Measurement(iterations = 25)
+@Warmup(iterations = 5)
+@Measurement(iterations = 15)
 public class TopNBenchmark
 {
   @Param({"750000"})
@@ -111,6 +111,9 @@ public class TopNBenchmark
   @Param({"10"})
   private int threshold;
 
+  @Param({"all", "hour"})
+  private String queryGranularity;
+
   private static final Logger log = new Logger(TopNBenchmark.class);
   private static final int RNG_SEED = 9999;
   private static final IndexMergerV9 INDEX_MERGER_V9;
@@ -159,7 +162,7 @@ public class TopNBenchmark
 
       TopNQueryBuilder queryBuilderA = new TopNQueryBuilder()
           .dataSource("blah")
-          .granularity(Granularities.ALL)
+          .granularity(Granularity.fromString(queryGranularity))
           .dimension("dimSequential")
           .metric("sumFloatNormal")
           .intervals(intervalSpec)
@@ -175,7 +178,7 @@ public class TopNBenchmark
 
       TopNQueryBuilder queryBuilderA = new TopNQueryBuilder()
           .dataSource("blah")
-          .granularity(Granularities.ALL)
+          .granularity(Granularity.fromString(queryGranularity))
           .dimension("dimUniform")
           .metric(new DimensionTopNMetricSpec(null, StringComparators.NUMERIC))
           .intervals(intervalSpec)
@@ -191,7 +194,7 @@ public class TopNBenchmark
 
       TopNQueryBuilder queryBuilderA = new TopNQueryBuilder()
           .dataSource("blah")
-          .granularity(Granularities.ALL)
+          .granularity(Granularity.fromString(queryGranularity))
           .dimension("dimUniform")
           .metric(new DimensionTopNMetricSpec(null, StringComparators.ALPHANUMERIC))
           .intervals(intervalSpec)
@@ -249,7 +252,7 @@ public class TopNBenchmark
   @State(Scope.Benchmark)
   public static class IncrementalIndexState
   {
-    @Param({"onheap", "offheap"})
+    @Param({"onheap"})
     private String indexType;
 
     IncrementalIndex incIndex;
diff --git a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalFactoryTestBase.java b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalFactoryTestBase.java
index 3562e6a35d6..ca98eb3796f 100644
--- a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalFactoryTestBase.java
+++ b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalFactoryTestBase.java
@@ -22,7 +22,6 @@ package org.apache.druid.compressedbigdecimal;
 import com.fasterxml.jackson.databind.MapperFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
-import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.query.aggregation.AggregateCombiner;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.query.aggregation.AggregatorFactory;
@@ -151,8 +150,7 @@ public abstract class CompressedBigDecimalFactoryTestBase
   {
     TestColumnValueSelector<CompressedBigDecimal> columnValueSelector = TestColumnValueSelector.of(
         CompressedBigDecimal.class,
-        ImmutableList.of(ArrayCompressedBigDecimal.wrap(new int[]{67, 0}, 0)),
-        DateTimes.of("2020-01-01")
+        ImmutableList.of(ArrayCompressedBigDecimal.wrap(new int[]{67, 0}, 0))
     );
 
     columnValueSelector.advance();
@@ -167,8 +165,7 @@ public abstract class CompressedBigDecimalFactoryTestBase
         ImmutableList.of(
             ArrayCompressedBigDecimal.wrap(new int[]{1, 0}, 0),
             ArrayCompressedBigDecimal.wrap(new int[]{10, 0}, 0)
-        ),
-        DateTimes.of("2020-01-01")
+        )
     );
 
     columnValueSelector.advance();
diff --git a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/MovingAverageQuery.java b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/MovingAverageQuery.java
index 280bc8ccceb..cbd9272dd8d 100644
--- a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/MovingAverageQuery.java
+++ b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/MovingAverageQuery.java
@@ -103,7 +103,7 @@ public class MovingAverageQuery extends BaseQuery<Row>
       @JsonProperty("context") Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec, false, context);
+    super(dataSource, querySegmentSpec, context);
 
     //TBD: Implement null awareness to respect the contract of this flag.
     Preconditions.checkArgument(
@@ -207,7 +207,7 @@ public class MovingAverageQuery extends BaseQuery<Row>
       Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec, false, context);
+    super(dataSource, querySegmentSpec, context);
 
     this.dimFilter = dimFilter;
     this.granularity = granularity;
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/util/ToObjectVectorColumnProcessorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/util/ToObjectVectorColumnProcessorFactoryTest.java
index e147cbc0377..b69b1fe8861 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/util/ToObjectVectorColumnProcessorFactoryTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/util/ToObjectVectorColumnProcessorFactoryTest.java
@@ -19,15 +19,18 @@
 
 package org.apache.druid.query.aggregation.datasketches.util;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import org.apache.druid.hll.HyperLogLogCollector;
-import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.query.QueryContexts;
 import org.apache.druid.segment.ColumnProcessors;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.TestIndex;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.vector.VectorCursor;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.hamcrest.CoreMatchers;
@@ -54,7 +57,8 @@ public class ToObjectVectorColumnProcessorFactoryTest extends InitializedNullHan
   @Test
   public void testRead()
   {
-    try (final VectorCursor cursor = makeCursor()) {
+    try (final CursorHolder cursorHolder = makeCursorHolder()) {
+      final VectorCursor cursor = cursorHolder.asVectorCursor();
       final Supplier<Object[]> qualitySupplier = ColumnProcessors.makeVectorProcessor(
           "quality",
           ToObjectVectorColumnProcessorFactory.INSTANCE,
@@ -167,21 +171,22 @@ public class ToObjectVectorColumnProcessorFactoryTest extends InitializedNullHan
     Assert.assertThat(sketch, CoreMatchers.instanceOf(HyperLogLogCollector.class));
   }
 
-  private VectorCursor makeCursor()
+  private CursorHolder makeCursorHolder()
   {
-    return adapter.makeVectorCursor(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        false,
-        3, /* vector size */
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setQueryContext(
+                                                         QueryContext.of(
+                                                             ImmutableMap.of(QueryContexts.VECTOR_SIZE_KEY, 3)
+                                                         )
+                                                     )
+                                                     .build();
+    return adapter.makeCursorHolder(buildSpec);
   }
 
   private List<Object> readColumn(final String column, final int limit)
   {
-    try (final VectorCursor cursor = makeCursor()) {
+    try (final CursorHolder cursorHolder = makeCursorHolder()) {
+      final VectorCursor cursor = cursorHolder.asVectorCursor();
       final Supplier<Object[]> supplier = ColumnProcessors.makeVectorProcessor(
           column,
           ToObjectVectorColumnProcessorFactory.INSTANCE,
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
index 3abf4b59c7d..aeb91a6c253 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
@@ -47,6 +47,8 @@ import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.msq.indexing.destination.DataSourceMSQDestination;
 import org.apache.druid.msq.util.MultiStageQueryContext;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryContext;
 import org.apache.druid.query.QueryContexts;
@@ -60,7 +62,6 @@ import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
 import org.apache.druid.query.groupby.orderby.OrderByColumnSpec;
-import org.apache.druid.query.scan.ScanQuery;
 import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
@@ -382,9 +383,9 @@ public class MSQCompactionRunner implements CompactionRunner
           orderByColumnSpecs
               .stream()
               .map(orderByColumnSpec ->
-                       new ScanQuery.OrderBy(
+                       new OrderBy(
                            orderByColumnSpec.getDimension(),
-                           ScanQuery.Order.fromString(orderByColumnSpec.getDirection().toString())
+                           Order.fromString(orderByColumnSpec.getDirection().toString())
                        ))
               .collect(Collectors.toList())
       );
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/processor/SegmentGeneratorFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/processor/SegmentGeneratorFrameProcessor.java
index 78b3e16f670..a4b52abee58 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/processor/SegmentGeneratorFrameProcessor.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/processor/SegmentGeneratorFrameProcessor.java
@@ -38,17 +38,16 @@ import org.apache.druid.frame.read.FrameReader;
 import org.apache.druid.frame.segment.FrameStorageAdapter;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.msq.counters.SegmentGenerationProgressCounter;
 import org.apache.druid.msq.exec.MSQTasks;
 import org.apache.druid.msq.input.ReadableInput;
-import org.apache.druid.msq.util.SequenceUtils;
 import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.realtime.appenderator.Appenderator;
@@ -182,42 +181,41 @@ public class SegmentGeneratorFrameProcessor implements FrameProcessor<DataSegmen
     // Reuse input row to avoid redoing allocations.
     final MSQInputRow inputRow = new MSQInputRow();
 
-    final Sequence<Cursor> cursorSequence =
-        new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY)
-            .makeCursors(null, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null);
+    final StorageAdapter adapter = new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY);
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      if (cursor == null) {
+        return;
+      }
 
-    SequenceUtils.forEach(
-        cursorSequence,
-        cursor -> {
-          final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+      final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-          //noinspection rawtypes
-          @SuppressWarnings("rawtypes")
-          final List<BaseObjectColumnValueSelector> selectors =
-              frameReader.signature()
-                         .getColumnNames()
-                         .stream()
-                         .map(columnSelectorFactory::makeColumnValueSelector)
-                         .collect(Collectors.toList());
+      //noinspection rawtypes
+      @SuppressWarnings("rawtypes")
+      final List<BaseObjectColumnValueSelector> selectors =
+          frameReader.signature()
+                     .getColumnNames()
+                     .stream()
+                     .map(columnSelectorFactory::makeColumnValueSelector)
+                     .collect(Collectors.toList());
 
-          while (!cursor.isDone()) {
-            for (int j = 0; j < signature.size(); j++) {
-              inputRow.getBackingArray()[j] = selectors.get(j).getObject();
-            }
-
-            try {
-              rowsWritten++;
-              appenderator.add(segmentIdWithShardSpec, inputRow, null);
-              segmentGenerationProgressCounter.incrementRowsProcessed(1);
-            }
-            catch (Exception e) {
-              throw new RuntimeException(e);
-            }
-
-            cursor.advance();
-          }
+      while (!cursor.isDone()) {
+        for (int j = 0; j < signature.size(); j++) {
+          inputRow.getBackingArray()[j] = selectors.get(j).getObject();
         }
-    );
+
+        try {
+          rowsWritten++;
+          appenderator.add(segmentIdWithShardSpec, inputRow, null);
+          segmentGenerationProgressCounter.incrementRowsProcessed(1);
+        }
+        catch (Exception e) {
+          throw new RuntimeException(e);
+        }
+
+        cursor.advance();
+      }
+    }
   }
 
   private class MSQInputRow implements InputRow
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/results/ExportResultsFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/results/ExportResultsFrameProcessor.java
index ccedf0402c6..665cfab89ee 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/results/ExportResultsFrameProcessor.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/results/ExportResultsFrameProcessor.java
@@ -33,15 +33,14 @@ import org.apache.druid.frame.processor.ReturnOrAwait;
 import org.apache.druid.frame.read.FrameReader;
 import org.apache.druid.frame.segment.FrameStorageAdapter;
 import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.msq.counters.ChannelCounters;
 import org.apache.druid.msq.exec.ResultsContext;
-import org.apache.druid.msq.util.SequenceUtils;
 import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.sql.calcite.planner.ColumnMapping;
 import org.apache.druid.sql.calcite.planner.ColumnMappings;
@@ -152,57 +151,58 @@ public class ExportResultsFrameProcessor implements FrameProcessor<Object>
 
   private void exportFrame(final Frame frame)
   {
-    final Sequence<Cursor> cursorSequence =
-        new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY)
-            .makeCursors(null, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null);
+    final StorageAdapter adapter = new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY);
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      if (cursor == null) {
+        exportWriter.writeRowEnd();
+        return;
+      }
+      final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    SequenceUtils.forEach(
-        cursorSequence,
-        cursor -> {
-          try {
-            final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+      //noinspection rawtypes
+      final List<BaseObjectColumnValueSelector> selectors =
+          frameReader.signature()
+                     .getColumnNames()
+                     .stream()
+                     .map(columnSelectorFactory::makeColumnValueSelector)
+                     .collect(Collectors.toList());
 
-            //noinspection rawtypes
-            final List<BaseObjectColumnValueSelector> selectors =
-                frameReader.signature()
-                           .getColumnNames()
-                           .stream()
-                           .map(columnSelectorFactory::makeColumnValueSelector)
-                           .collect(Collectors.toList());
-
-            while (!cursor.isDone()) {
-              exportWriter.writeRowStart();
-              for (int j = 0; j < exportRowSignature.size(); j++) {
-                String columnName = exportRowSignature.getColumnName(j);
-                BaseObjectColumnValueSelector<?> selector = selectors.get(outputColumnNameToFrameColumnNumberMap.getInt(columnName));
-                if (resultsContext == null) {
-                  throw DruidException.forPersona(DruidException.Persona.OPERATOR)
-                                      .ofCategory(DruidException.Category.RUNTIME_FAILURE)
-                                      .build("Received null resultsContext from the controller. This is due to a version mismatch between the controller and the worker. Please ensure that the worker and the controller are on the same version before retrying the query.");
-                }
-                exportWriter.writeRowField(
-                    columnName,
-                    SqlResults.coerce(
-                        jsonMapper,
-                        resultsContext.getSqlResultsContext(),
-                        selector.getObject(),
-                        resultsContext.getSqlTypeNames().get(j),
-                        columnName
-                    )
-                );
-              }
-              channelCounter.incrementRowCount(partitionNum);
-              exportWriter.writeRowEnd();
-              cursor.advance();
-            }
-          }
-          catch (IOException e) {
-            throw DruidException.forPersona(DruidException.Persona.USER)
+      while (!cursor.isDone()) {
+        exportWriter.writeRowStart();
+        for (int j = 0; j < exportRowSignature.size(); j++) {
+          String columnName = exportRowSignature.getColumnName(j);
+          BaseObjectColumnValueSelector<?> selector = selectors.get(outputColumnNameToFrameColumnNumberMap.getInt(columnName));
+          if (resultsContext == null) {
+            throw DruidException.forPersona(DruidException.Persona.OPERATOR)
                                 .ofCategory(DruidException.Category.RUNTIME_FAILURE)
-                                .build(e, "Exception occurred while writing file to the export location [%s].", exportFilePath);
+                                .build("Received null resultsContext from the controller. This is due to a version mismatch between the controller and the worker. Please ensure that the worker and the controller are on the same version before retrying the query.");
           }
+          exportWriter.writeRowField(
+              columnName,
+              SqlResults.coerce(
+                  jsonMapper,
+                  resultsContext.getSqlResultsContext(),
+                  selector.getObject(),
+                  resultsContext.getSqlTypeNames().get(j),
+                  columnName
+              )
+          );
         }
-    );
+        channelCounter.incrementRowCount(partitionNum);
+        exportWriter.writeRowEnd();
+        cursor.advance();
+      }
+    }
+    catch (IOException e) {
+      throw DruidException.forPersona(DruidException.Persona.USER)
+                          .ofCategory(DruidException.Category.RUNTIME_FAILURE)
+                          .build(
+                              e,
+                              "Exception occurred while writing file to the export location [%s].",
+                              exportFilePath
+                          );
+    }
   }
 
   private void createExportWriter() throws IOException
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessor.java
index 2104d1d40f4..f402aa60430 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessor.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessor.java
@@ -20,7 +20,6 @@
 package org.apache.druid.msq.querykit.scan;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
@@ -41,10 +40,8 @@ import org.apache.druid.frame.write.FrameWriterFactory;
 import org.apache.druid.frame.write.InvalidFieldException;
 import org.apache.druid.frame.write.InvalidNullByteException;
 import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.Unit;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.guava.Yielder;
@@ -62,13 +59,13 @@ import org.apache.druid.msq.querykit.BaseLeafFrameProcessor;
 import org.apache.druid.msq.querykit.QueryKitUtils;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.IterableRowsCursorHelper;
-import org.apache.druid.query.filter.Filter;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.scan.ScanQuery;
+import org.apache.druid.query.scan.ScanQueryEngine;
 import org.apache.druid.query.scan.ScanResultValue;
-import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
-import org.apache.druid.query.spec.SpecificSegmentSpec;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.SegmentReference;
 import org.apache.druid.segment.SimpleAscendingOffset;
@@ -77,9 +74,7 @@ import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumn;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.RowSignature;
-import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.timeline.SegmentId;
-import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
@@ -180,7 +175,7 @@ public class ScanQueryFrameProcessor extends BaseLeafFrameProcessor
    */
   private static ScanQuery prepareScanQueryForDataServer(@NotNull ScanQuery scanQuery)
   {
-    if (ScanQuery.Order.NONE.equals(scanQuery.getTimeOrder()) && !scanQuery.getOrderBys().isEmpty()) {
+    if (Order.NONE.equals(scanQuery.getTimeOrder()) && !scanQuery.getOrderBys().isEmpty()) {
       return Druids.ScanQueryBuilder.copy(scanQuery)
                                     .orderBy(ImmutableList.of())
                                     .limit(0)
@@ -252,21 +247,24 @@ public class ScanQueryFrameProcessor extends BaseLeafFrameProcessor
     if (cursor == null) {
       final ResourceHolder<Segment> segmentHolder = closer.register(segment.getOrLoad());
 
-      final Yielder<Cursor> cursorYielder = Yielders.each(
-          makeCursors(
-              query.withQuerySegmentSpec(new SpecificSegmentSpec(segment.getDescriptor())),
-              mapSegment(segmentHolder.get()).asStorageAdapter()
-          )
-      );
+      final StorageAdapter adapter = mapSegment(segmentHolder.get()).asStorageAdapter();
+      if (adapter == null) {
+        throw new ISE(
+            "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped."
+        );
+      }
 
-      if (cursorYielder.isDone()) {
+      final CursorHolder cursorHolder = closer.register(
+          adapter.makeCursorHolder(ScanQueryEngine.makeCursorBuildSpec(query, null))
+      );
+      final Cursor nextCursor = cursorHolder.asCursor();
+
+      if (nextCursor == null) {
         // No cursors!
-        cursorYielder.close();
         return ReturnOrAwait.returnObject(Unit.instance());
       } else {
-        final long rowsFlushed = setNextCursor(cursorYielder.get(), segmentHolder.get());
+        final long rowsFlushed = setNextCursor(nextCursor, segmentHolder.get());
         assert rowsFlushed == 0; // There's only ever one cursor when running with a segment
-        closer.register(cursorYielder);
       }
     }
 
@@ -294,15 +292,23 @@ public class ScanQueryFrameProcessor extends BaseLeafFrameProcessor
         final Frame frame = inputChannel.read();
         final FrameSegment frameSegment = new FrameSegment(frame, inputFrameReader, SegmentId.dummy("scan"));
 
-        final long rowsFlushed = setNextCursor(
-            Iterables.getOnlyElement(
-                makeCursors(
-                    query.withQuerySegmentSpec(new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY)),
-                    mapSegment(frameSegment).asStorageAdapter()
-                ).toList()
-            ),
-            frameSegment
+        final StorageAdapter adapter = mapSegment(frameSegment).asStorageAdapter();
+        if (adapter == null) {
+          throw new ISE(
+              "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped."
+          );
+        }
+
+        final CursorHolder cursorHolder = closer.register(
+            adapter.makeCursorHolder(ScanQueryEngine.makeCursorBuildSpec(query, null))
         );
+        final Cursor nextCursor = cursorHolder.asCursor();
+
+        if (nextCursor == null) {
+          // no cursor
+          return ReturnOrAwait.returnObject(Unit.instance());
+        }
+        final long rowsFlushed = setNextCursor(nextCursor, frameSegment);
 
         if (rowsFlushed > 0) {
           return ReturnOrAwait.runAgain();
@@ -430,27 +436,4 @@ public class ScanQueryFrameProcessor extends BaseLeafFrameProcessor
     }
     return baseColumnSelectorFactory;
   }
-
-  private static Sequence<Cursor> makeCursors(final ScanQuery query, final StorageAdapter adapter)
-  {
-    if (adapter == null) {
-      throw new ISE(
-          "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped."
-      );
-    }
-
-    final List<Interval> intervals = query.getQuerySegmentSpec().getIntervals();
-    Preconditions.checkArgument(intervals.size() == 1, "Can only handle a single interval, got[%s]", intervals);
-
-    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter()));
-
-    return adapter.makeCursors(
-        filter,
-        intervals.get(0),
-        query.getVirtualColumns(),
-        Granularities.ALL,
-        ScanQuery.Order.DESCENDING.equals(query.getTimeOrder()),
-        null
-    );
-  }
 }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryKit.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryKit.java
index 2a90616fe1d..5bbf9c9cbb0 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryKit.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryKit.java
@@ -37,6 +37,8 @@ import org.apache.druid.msq.querykit.QueryKitUtils;
 import org.apache.druid.msq.querykit.ShuffleSpecFactories;
 import org.apache.druid.msq.querykit.ShuffleSpecFactory;
 import org.apache.druid.msq.querykit.common.OffsetLimitFrameProcessorFactory;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.scan.ScanQuery;
 import org.apache.druid.segment.column.ColumnType;
@@ -119,11 +121,11 @@ public class ScanQueryKit implements QueryKit<ScanQuery>
     final List<KeyColumn> clusterByColumns = new ArrayList<>();
 
     // Add regular orderBys.
-    for (final ScanQuery.OrderBy orderBy : queryToRun.getOrderBys()) {
+    for (final OrderBy orderBy : queryToRun.getOrderBys()) {
       clusterByColumns.add(
           new KeyColumn(
               orderBy.getColumnName(),
-              orderBy.getOrder() == ScanQuery.Order.DESCENDING ? KeyOrder.DESCENDING : KeyOrder.ASCENDING
+              orderBy.getOrder() == Order.DESCENDING ? KeyOrder.DESCENDING : KeyOrder.ASCENDING
           )
       );
     }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java
index 20a871b953c..03890d7a326 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java
@@ -36,9 +36,9 @@ import org.apache.druid.msq.test.MSQTestBase;
 import org.apache.druid.msq.util.MultiStageQueryContext;
 import org.apache.druid.query.DataSource;
 import org.apache.druid.query.NestedDataTestUtils;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.expression.TestExprMacroTable;
-import org.apache.druid.query.scan.ScanQuery;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
@@ -862,7 +862,7 @@ public class MSQArraysTest extends MSQTestBase
         .dataSource(dataFileExternalDataSource)
         .intervals(querySegmentSpec(Filtration.eternity()))
         .columns("arrayString")
-        .orderBy(Collections.singletonList(new ScanQuery.OrderBy("arrayString", ScanQuery.Order.DESCENDING)))
+        .orderBy(Collections.singletonList(OrderBy.descending("arrayString")))
         .context(defaultScanQueryContext(context, scanSignature))
         .build();
 
@@ -925,7 +925,7 @@ public class MSQArraysTest extends MSQTestBase
         .dataSource(dataFileExternalDataSource)
         .intervals(querySegmentSpec(Filtration.eternity()))
         .columns("arrayLong")
-        .orderBy(Collections.singletonList(new ScanQuery.OrderBy("arrayLong", ScanQuery.Order.ASCENDING)))
+        .orderBy(Collections.singletonList(OrderBy.ascending("arrayLong")))
         .context(defaultScanQueryContext(context, scanSignature))
         .build();
 
@@ -988,7 +988,7 @@ public class MSQArraysTest extends MSQTestBase
         .dataSource(dataFileExternalDataSource)
         .intervals(querySegmentSpec(Filtration.eternity()))
         .columns("arrayDouble")
-        .orderBy(Collections.singletonList(new ScanQuery.OrderBy("arrayDouble", ScanQuery.Order.ASCENDING)))
+        .orderBy(Collections.singletonList(OrderBy.ascending("arrayDouble")))
         .context(defaultScanQueryContext(context, scanSignature))
         .build();
 
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
index f937a988d8e..e498b5bb268 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
@@ -36,6 +36,7 @@ import org.apache.druid.msq.test.MSQTestBase;
 import org.apache.druid.msq.util.MultiStageQueryContext;
 import org.apache.druid.query.DataSource;
 import org.apache.druid.query.NestedDataTestUtils;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
@@ -50,7 +51,6 @@ import org.apache.druid.query.groupby.GroupByQueryConfig;
 import org.apache.druid.query.groupby.orderby.DefaultLimitSpec;
 import org.apache.druid.query.groupby.orderby.OrderByColumnSpec;
 import org.apache.druid.query.ordering.StringComparators;
-import org.apache.druid.query.scan.ScanQuery;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.nested.StructuredData;
@@ -410,7 +410,7 @@ public class MSQComplexGroupByTest extends MSQTestBase
                                                         .intervals(querySegmentSpec(Filtration.eternity()))
                                                         .columns("obj")
                                                         .context(defaultScanQueryContext(context, rowSignature))
-                                                        .orderBy(Collections.singletonList(new ScanQuery.OrderBy("obj", ScanQuery.Order.ASCENDING)))
+                                                        .orderBy(Collections.singletonList(OrderBy.ascending("obj")))
                                                         .build()
                                              )
                                              .columnMappings(new ColumnMappings(ImmutableList.of(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQLoadedSegmentTests.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQLoadedSegmentTests.java
index ab06f851af0..4ea4cb0344f 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQLoadedSegmentTests.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQLoadedSegmentTests.java
@@ -33,6 +33,7 @@ import org.apache.druid.msq.indexing.MSQTuningConfig;
 import org.apache.druid.msq.indexing.destination.TaskReportMSQDestination;
 import org.apache.druid.msq.test.MSQTestBase;
 import org.apache.druid.msq.util.MultiStageQueryContext;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.groupby.GroupByQuery;
@@ -189,7 +190,7 @@ public class MSQLoadedSegmentTests extends MSQTestBase
                            .dataSource(CalciteTests.DATASOURCE1)
                            .intervals(querySegmentSpec(Filtration.eternity()))
                            .columns("cnt", "dim1")
-                           .orderBy(ImmutableList.of(new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING)))
+                           .orderBy(ImmutableList.of(OrderBy.ascending("dim1")))
                            .context(defaultScanQueryContext(REALTIME_QUERY_CTX, resultSignature))
                            .build()
                    )
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
index c6d67f14428..b80b3844b50 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
@@ -49,6 +49,7 @@ import org.apache.druid.msq.test.MSQTestBase;
 import org.apache.druid.msq.util.MultiStageQueryContext;
 import org.apache.druid.query.InlineDataSource;
 import org.apache.druid.query.LookupDataSource;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.TableDataSource;
@@ -463,7 +464,7 @@ public class MSQSelectTest extends MSQTestBase
                            .columns("cnt", "dim1")
                            .filters(equality("dim2", "nonexistent", ColumnType.STRING))
                            .context(defaultScanQueryContext(context, resultSignature))
-                           .orderBy(ImmutableList.of(new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING)))
+                           .orderBy(ImmutableList.of(OrderBy.ascending("dim1")))
                            .build()
                    )
                    .columnMappings(ColumnMappings.identity(resultSignature))
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java
index 3fbd8c72049..ee7121b3678 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java
@@ -32,6 +32,7 @@ import org.apache.druid.msq.test.CounterSnapshotMatcher;
 import org.apache.druid.msq.test.MSQTestBase;
 import org.apache.druid.msq.util.MultiStageQueryContext;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.TableDataSource;
@@ -2135,9 +2136,9 @@ public class MSQWindowTest extends MSQTestBase
                                   .columns("d0", "d1", "d2", "w0", "w1")
                                   .orderBy(
                                       ImmutableList.of(
-                                          new ScanQuery.OrderBy("d0", ScanQuery.Order.ASCENDING),
-                                          new ScanQuery.OrderBy("d1", ScanQuery.Order.ASCENDING),
-                                          new ScanQuery.OrderBy("d2", ScanQuery.Order.ASCENDING)
+                                          OrderBy.ascending("d0"),
+                                          OrderBy.ascending("d1"),
+                                          OrderBy.ascending("d2")
                                       )
                                   )
                                   .columnTypes(ColumnType.STRING, ColumnType.STRING, ColumnType.STRING, ColumnType.LONG, ColumnType.LONG)
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
index 9c4bb91637c..94be3495b98 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
@@ -48,6 +48,7 @@ import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.msq.indexing.destination.DataSourceMSQDestination;
 import org.apache.druid.msq.kernel.WorkerAssignmentStrategy;
 import org.apache.druid.msq.util.MultiStageQueryContext;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@@ -301,10 +302,10 @@ public class MSQCompactionRunnerTest
     );
     Assert.assertNull(msqControllerTask.getContext().get(DruidSqlInsert.SQL_INSERT_QUERY_GRANULARITY));
     Assert.assertEquals(WorkerAssignmentStrategy.MAX, actualMSQSpec.getAssignmentStrategy());
-    Assert.assertEquals(PARTITION_DIMENSIONS.stream().map(col -> new ScanQuery.OrderBy(
-        col,
-        ScanQuery.Order.ASCENDING
-    )).collect(Collectors.toList()), ((ScanQuery) actualMSQSpec.getQuery()).getOrderBys());
+    Assert.assertEquals(
+        PARTITION_DIMENSIONS.stream().map(OrderBy::ascending).collect(Collectors.toList()),
+        ((ScanQuery) actualMSQSpec.getQuery()).getOrderBys()
+    );
   }
 
   @Test
diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/hadoop/DatasourceRecordReader.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/hadoop/DatasourceRecordReader.java
index 40a67f1236e..d1ceccc96a3 100644
--- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/hadoop/DatasourceRecordReader.java
+++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/hadoop/DatasourceRecordReader.java
@@ -33,7 +33,6 @@ import org.apache.druid.indexer.HadoopDruidIndexerConfig;
 import org.apache.druid.indexer.JobHelper;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.FileUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.guava.Yielder;
@@ -44,10 +43,11 @@ import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.segment.BaseLongColumnValueSelector;
 import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.data.IndexedInts;
 import org.apache.druid.segment.filter.Filters;
@@ -214,108 +214,98 @@ public class DatasourceRecordReader extends RecordReader<NullWritable, InputRow>
                 @Override
                 public Sequence<InputRow> apply(WindowedStorageAdapter adapter)
                 {
-                  return Sequences.concat(
-                      Sequences.map(
-                          adapter.getAdapter().makeCursors(
-                              Filters.toFilter(dimFilter),
-                              adapter.getInterval(),
-                              VirtualColumns.EMPTY,
-                              Granularities.ALL,
-                              false,
-                              null
-                          ), new Function<Cursor, Sequence<InputRow>>()
-                          {
-                            @Nullable
-                            @Override
-                            public Sequence<InputRow> apply(final Cursor cursor)
-                            {
-                              final BaseLongColumnValueSelector timestampColumnSelector =
-                                  cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME);
+                  final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                                   .setFilter(Filters.toFilter(dimFilter))
+                                                                   .setInterval(adapter.getInterval())
+                                                                   .build();
+                  final CursorHolder cursorHolder = adapter.getAdapter().makeCursorHolder(buildSpec);
+                  final Cursor cursor = cursorHolder.asCursor();
+                  if (cursor == null) {
+                    return Sequences.empty();
+                  }
+                  final BaseLongColumnValueSelector timestampColumnSelector =
+                      cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME);
 
-                              final Map<String, DimensionSelector> dimSelectors = new HashMap<>();
-                              for (String dim : dims) {
-                                final DimensionSelector dimSelector = cursor
-                                    .getColumnSelectorFactory()
-                                    .makeDimensionSelector(new DefaultDimensionSpec(dim, dim));
-                                // dimSelector is null if the dimension is not present
-                                if (dimSelector != null) {
-                                  dimSelectors.put(dim, dimSelector);
+                  final Map<String, DimensionSelector> dimSelectors = new HashMap<>();
+                  for (String dim : dims) {
+                    final DimensionSelector dimSelector = cursor
+                        .getColumnSelectorFactory()
+                        .makeDimensionSelector(new DefaultDimensionSpec(dim, dim));
+                    // dimSelector is null if the dimension is not present
+                    if (dimSelector != null) {
+                      dimSelectors.put(dim, dimSelector);
+                    }
+                  }
+
+                  final Map<String, BaseObjectColumnValueSelector> metSelectors = new HashMap<>();
+                  for (String metric : metrics) {
+                    final BaseObjectColumnValueSelector metricSelector =
+                        cursor.getColumnSelectorFactory().makeColumnValueSelector(metric);
+                    metSelectors.put(metric, metricSelector);
+                  }
+
+                  return Sequences.simple(
+                      new Iterable<InputRow>()
+                      {
+                        @Override
+                        public Iterator<InputRow> iterator()
+                        {
+                          return new Iterator<InputRow>()
+                          {
+                            @Override
+                            public boolean hasNext()
+                            {
+                              return !cursor.isDone();
+                            }
+
+                            @Override
+                            public InputRow next()
+                            {
+                              final Map<String, Object> theEvent = Maps.newLinkedHashMap();
+                              final long timestamp = timestampColumnSelector.getLong();
+                              theEvent.put(TimestampSpec.DEFAULT_COLUMN, DateTimes.utc(timestamp));
+
+                              for (Map.Entry<String, DimensionSelector> dimSelector :
+                                  dimSelectors.entrySet()) {
+                                final String dim = dimSelector.getKey();
+                                final DimensionSelector selector = dimSelector.getValue();
+                                final IndexedInts vals = selector.getRow();
+
+                                int valsSize = vals.size();
+                                if (valsSize == 1) {
+                                  final String dimVal = selector.lookupName(vals.get(0));
+                                  theEvent.put(dim, dimVal);
+                                } else if (valsSize > 1) {
+                                  List<String> dimVals = new ArrayList<>(valsSize);
+                                  for (int i = 0; i < valsSize; ++i) {
+                                    dimVals.add(selector.lookupName(vals.get(i)));
+                                  }
+                                  theEvent.put(dim, dimVals);
                                 }
                               }
 
-                              final Map<String, BaseObjectColumnValueSelector> metSelectors = new HashMap<>();
-                              for (String metric : metrics) {
-                                final BaseObjectColumnValueSelector metricSelector =
-                                    cursor.getColumnSelectorFactory().makeColumnValueSelector(metric);
-                                metSelectors.put(metric, metricSelector);
+                              for (Map.Entry<String, BaseObjectColumnValueSelector> metSelector :
+                                  metSelectors.entrySet()) {
+                                final String metric = metSelector.getKey();
+                                final BaseObjectColumnValueSelector selector = metSelector.getValue();
+                                Object value = selector.getObject();
+                                if (value != null) {
+                                  theEvent.put(metric, value);
+                                }
                               }
-
-                              return Sequences.simple(
-                                  new Iterable<InputRow>()
-                                  {
-                                    @Override
-                                    public Iterator<InputRow> iterator()
-                                    {
-                                      return new Iterator<InputRow>()
-                                      {
-                                        @Override
-                                        public boolean hasNext()
-                                        {
-                                          return !cursor.isDone();
-                                        }
-
-                                        @Override
-                                        public InputRow next()
-                                        {
-                                          final Map<String, Object> theEvent = Maps.newLinkedHashMap();
-                                          final long timestamp = timestampColumnSelector.getLong();
-                                          theEvent.put(TimestampSpec.DEFAULT_COLUMN, DateTimes.utc(timestamp));
-
-                                          for (Map.Entry<String, DimensionSelector> dimSelector :
-                                              dimSelectors.entrySet()) {
-                                            final String dim = dimSelector.getKey();
-                                            final DimensionSelector selector = dimSelector.getValue();
-                                            final IndexedInts vals = selector.getRow();
-
-                                            int valsSize = vals.size();
-                                            if (valsSize == 1) {
-                                              final String dimVal = selector.lookupName(vals.get(0));
-                                              theEvent.put(dim, dimVal);
-                                            } else if (valsSize > 1) {
-                                              List<String> dimVals = new ArrayList<>(valsSize);
-                                              for (int i = 0; i < valsSize; ++i) {
-                                                dimVals.add(selector.lookupName(vals.get(i)));
-                                              }
-                                              theEvent.put(dim, dimVals);
-                                            }
-                                          }
-
-                                          for (Map.Entry<String, BaseObjectColumnValueSelector> metSelector :
-                                              metSelectors.entrySet()) {
-                                            final String metric = metSelector.getKey();
-                                            final BaseObjectColumnValueSelector selector = metSelector.getValue();
-                                            Object value = selector.getObject();
-                                            if (value != null) {
-                                              theEvent.put(metric, value);
-                                            }
-                                          }
-                                          cursor.advance();
-                                          return new MapBasedInputRow(timestamp, dims, theEvent);
-                                        }
-
-                                        @Override
-                                        public void remove()
-                                        {
-                                          throw new UnsupportedOperationException("Remove Not Supported");
-                                        }
-                                      };
-                                    }
-                                  }
-                              );
+                              cursor.advance();
+                              return new MapBasedInputRow(timestamp, dims, theEvent);
                             }
-                          }
-                      )
-                  );
+
+                            @Override
+                            public void remove()
+                            {
+                              throw new UnsupportedOperationException("Remove Not Supported");
+                            }
+                          };
+                        }
+                      }
+                  ).withBaggage(cursorHolder);
                 }
               }
           )
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/input/DruidSegmentReader.java b/indexing-service/src/main/java/org/apache/druid/indexing/input/DruidSegmentReader.java
index 42618556d00..a85fa73349d 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/input/DruidSegmentReader.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/input/DruidSegmentReader.java
@@ -33,7 +33,7 @@ import org.apache.druid.data.input.IntermediateRowParsingReader;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.MapInputRowParser;
 import org.apache.druid.data.input.impl.TimestampSpec;
-import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.CloseableIterators;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.guava.Yielder;
@@ -48,10 +48,11 @@ import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnProcessorFactory;
 import org.apache.druid.segment.ColumnProcessors;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.IndexedInts;
@@ -123,15 +124,16 @@ public class DruidSegmentReader extends IntermediateRowParsingReader<Map<String,
         ),
         intervalFilter
     );
+    final CursorBuildSpec cursorBuildSpec = CursorBuildSpec.builder()
+                                                           .setFilter(Filters.toFilter(dimFilter))
+                                                           .setInterval(storageAdapter.getInterval())
+                                                           .build();
 
-    final Sequence<Cursor> cursors = storageAdapter.getAdapter().makeCursors(
-        Filters.toFilter(dimFilter),
-        storageAdapter.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorHolder cursorHolder = storageAdapter.getAdapter().makeCursorHolder(cursorBuildSpec);
+    final Cursor cursor = cursorHolder.asCursor();
+    if (cursor == null) {
+      return CloseableIterators.wrap(Collections.emptyIterator(), cursorHolder);
+    }
 
     // Retain order of columns from the original segments. Useful for preserving dimension order if we're in
     // schemaless mode.
@@ -142,12 +144,7 @@ public class DruidSegmentReader extends IntermediateRowParsingReader<Map<String,
         )
     );
 
-    final Sequence<Map<String, Object>> sequence = Sequences.concat(
-        Sequences.map(
-            cursors,
-            cursor -> cursorToSequence(cursor, columnsToRead)
-        )
-    );
+    final Sequence<Map<String, Object>> sequence = cursorToSequence(cursor, columnsToRead).withBaggage(cursorHolder);
 
     return makeCloseableIteratorFromSequenceAndSegmentFile(sequence, segmentFile);
   }
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java
index 7ebde700bee..6fbf9e5deed 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java
@@ -61,7 +61,6 @@ import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Comparators;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@@ -72,12 +71,13 @@ import org.apache.druid.segment.AutoTypeColumnSchema;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DataSegmentsWithSchemas;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.TestIndex;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
 import org.apache.druid.segment.join.NoopJoinableFactory;
@@ -1702,23 +1702,19 @@ public class CompactionTaskRunTest extends IngestionTestBase
           new QueryableIndexStorageAdapter(testUtils.getTestIndexIO().loadIndex(segmentFile)),
           segment.getInterval()
       );
-      final Sequence<Cursor> cursorSequence = adapter.getAdapter().makeCursors(
-          null,
-          segment.getInterval(),
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
-
-      cursorSequence.accumulate(rowsFromSegment, (accumulated, cursor) -> {
+      try (final CursorHolder cursorHolder = adapter.getAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        final Cursor cursor = cursorHolder.asCursor();
+        Assert.assertNotNull(cursor);
         cursor.reset();
         final ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
         Assert.assertTrue(factory.getColumnCapabilities("spatial").hasSpatialIndexes());
         while (!cursor.isDone()) {
           final ColumnValueSelector<?> selector1 = factory.makeColumnValueSelector("ts");
           final DimensionSelector selector2 = factory.makeDimensionSelector(new DefaultDimensionSpec("dim", "dim"));
-          final DimensionSelector selector3 = factory.makeDimensionSelector(new DefaultDimensionSpec("spatial", "spatial"));
+          final DimensionSelector selector3 = factory.makeDimensionSelector(new DefaultDimensionSpec(
+              "spatial",
+              "spatial"
+          ));
           final DimensionSelector selector4 = factory.makeDimensionSelector(new DefaultDimensionSpec("val", "val"));
 
 
@@ -1734,9 +1730,7 @@ public class CompactionTaskRunTest extends IngestionTestBase
 
           cursor.advance();
         }
-
-        return accumulated;
-      });
+      }
     }
     Assert.assertEquals(spatialrows, rowsFromSegment);
   }
@@ -1836,16 +1830,9 @@ public class CompactionTaskRunTest extends IngestionTestBase
           new QueryableIndexStorageAdapter(testUtils.getTestIndexIO().loadIndex(segmentFile)),
           segment.getInterval()
       );
-      final Sequence<Cursor> cursorSequence = adapter.getAdapter().makeCursors(
-          null,
-          segment.getInterval(),
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
-
-      cursorSequence.accumulate(rowsFromSegment, (accumulated, cursor) -> {
+      try (final CursorHolder cursorHolder = adapter.getAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        final Cursor cursor = cursorHolder.asCursor();
+        Assert.assertNotNull(cursor);
         cursor.reset();
         final ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
         Assert.assertEquals(ColumnType.STRING, factory.getColumnCapabilities("ts").toColumnType());
@@ -1873,9 +1860,7 @@ public class CompactionTaskRunTest extends IngestionTestBase
 
           cursor.advance();
         }
-
-        return accumulated;
-      });
+      }
     }
     Assert.assertEquals(rows, rowsFromSegment);
   }
@@ -2070,16 +2055,9 @@ public class CompactionTaskRunTest extends IngestionTestBase
           new QueryableIndexStorageAdapter(testUtils.getTestIndexIO().loadIndex(segmentFile)),
           segment.getInterval()
       );
-      final Sequence<Cursor> cursorSequence = adapter.getAdapter().makeCursors(
-          null,
-          segment.getInterval(),
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
-
-      cursorSequence.accumulate(rowsFromSegment, (accumulated, cursor) -> {
+      try (final CursorHolder cursorHolder = adapter.getAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        final Cursor cursor = cursorHolder.asCursor();
+        Assert.assertNotNull(cursor);
         cursor.reset();
         while (!cursor.isDone()) {
           final DimensionSelector selector1 = cursor.getColumnSelectorFactory()
@@ -2107,9 +2085,7 @@ public class CompactionTaskRunTest extends IngestionTestBase
 
           cursor.advance();
         }
-
-        return accumulated;
-      });
+      }
     }
 
     return rowsFromSegment;
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java
index 3ec5b8d0aa8..c54446fc4df 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java
@@ -60,7 +60,6 @@ import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.metrics.StubServiceEmitter;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.aggregation.AggregatorFactory;
@@ -68,6 +67,8 @@ import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DataSegmentsWithSchemas;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IndexIO;
@@ -75,7 +76,6 @@ import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.SegmentSchemaMapping;
 import org.apache.druid.segment.TestIndex;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.CompressionStrategy;
@@ -540,71 +540,65 @@ public class IndexTaskTest extends IngestionTestBase
         new QueryableIndexStorageAdapter(indexIO.loadIndex(segmentFile)),
         segment.getInterval()
     );
-    final Sequence<Cursor> cursorSequence = adapter.getAdapter().makeCursors(
-        null,
-        segment.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-    final List<Map<String, Object>> transforms = cursorSequence
-        .map(cursor -> {
-          final DimensionSelector selector1 = cursor.getColumnSelectorFactory()
-                                                    .makeDimensionSelector(new DefaultDimensionSpec("dimt", "dimt"));
-          final DimensionSelector selector2 = cursor.getColumnSelectorFactory()
-                                                    .makeDimensionSelector(new DefaultDimensionSpec(
-                                                        "dimtarray1",
-                                                        "dimtarray1"
-                                                    ));
-          final DimensionSelector selector3 = cursor.getColumnSelectorFactory()
-                                                    .makeDimensionSelector(new DefaultDimensionSpec(
-                                                        "dimtarray2",
-                                                        "dimtarray2"
-                                                    ));
-          final DimensionSelector selector4 = cursor.getColumnSelectorFactory()
-                                                    .makeDimensionSelector(new DefaultDimensionSpec(
-                                                        "dimtnum_array",
-                                                        "dimtnum_array"
-                                                    ));
+    try (final CursorHolder cursorHolder = adapter.getAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final List<Map<String, Object>> transforms = new ArrayList<>();
+
+      final DimensionSelector selector1 = cursor.getColumnSelectorFactory()
+                                                .makeDimensionSelector(new DefaultDimensionSpec("dimt", "dimt"));
+      final DimensionSelector selector2 = cursor.getColumnSelectorFactory()
+                                                .makeDimensionSelector(new DefaultDimensionSpec(
+                                                    "dimtarray1",
+                                                    "dimtarray1"
+                                                ));
+      final DimensionSelector selector3 = cursor.getColumnSelectorFactory()
+                                                .makeDimensionSelector(new DefaultDimensionSpec(
+                                                    "dimtarray2",
+                                                    "dimtarray2"
+                                                ));
+      final DimensionSelector selector4 = cursor.getColumnSelectorFactory()
+                                                .makeDimensionSelector(new DefaultDimensionSpec(
+                                                    "dimtnum_array",
+                                                    "dimtnum_array"
+                                                ));
 
 
-          Map<String, Object> row = new HashMap<>();
-          row.put("dimt", selector1.defaultGetObject());
-          row.put("dimtarray1", selector2.defaultGetObject());
-          row.put("dimtarray2", selector3.defaultGetObject());
-          row.put("dimtnum_array", selector4.defaultGetObject());
-          cursor.advance();
-          return row;
-        })
-        .toList();
-    Assert.assertEquals(1, transforms.size());
-    Assert.assertEquals("bb", transforms.get(0).get("dimt"));
-    Assert.assertEquals(ImmutableList.of("b", "b"), transforms.get(0).get("dimtarray1"));
-    Assert.assertEquals(ImmutableList.of("anotherfoo", "arrayfoo"), transforms.get(0).get("dimtarray2"));
-    Assert.assertEquals(ImmutableList.of("6.0", "7.0"), transforms.get(0).get("dimtnum_array"));
+      Map<String, Object> row = new HashMap<>();
+      row.put("dimt", selector1.defaultGetObject());
+      row.put("dimtarray1", selector2.defaultGetObject());
+      row.put("dimtarray2", selector3.defaultGetObject());
+      row.put("dimtnum_array", selector4.defaultGetObject());
+      transforms.add(row);
+      cursor.advance();
 
-    Assert.assertEquals(DATASOURCE, segments.get(0).getDataSource());
-    Assert.assertEquals(Intervals.of("2014/P1D"), segments.get(0).getInterval());
-    Assert.assertEquals(NumberedShardSpec.class, segments.get(0).getShardSpec().getClass());
-    Assert.assertEquals(0, segments.get(0).getShardSpec().getPartitionNum());
+      Assert.assertEquals(1, transforms.size());
+      Assert.assertEquals("bb", transforms.get(0).get("dimt"));
+      Assert.assertEquals(ImmutableList.of("b", "b"), transforms.get(0).get("dimtarray1"));
+      Assert.assertEquals(ImmutableList.of("anotherfoo", "arrayfoo"), transforms.get(0).get("dimtarray2"));
+      Assert.assertEquals(ImmutableList.of("6.0", "7.0"), transforms.get(0).get("dimtnum_array"));
 
-    verifySchemaAndAggFactory(
-        segmentWithSchemas,
-        RowSignature.builder()
-                    .add("__time", ColumnType.LONG)
-                    .add("ts", ColumnType.STRING)
-                    .add("dim", ColumnType.STRING)
-                    .add("dim_array", ColumnType.STRING)
-                    .add("dim_num_array", ColumnType.STRING)
-                    .add("dimt", ColumnType.STRING)
-                    .add("dimtarray1", ColumnType.STRING)
-                    .add("dimtarray2", ColumnType.STRING)
-                    .add("dimtnum_array", ColumnType.STRING)
-                    .add("val", ColumnType.LONG)
-                    .build(),
-        Collections.singletonMap("val", new LongSumAggregatorFactory("val", "val"))
-    );
+      Assert.assertEquals(DATASOURCE, segments.get(0).getDataSource());
+      Assert.assertEquals(Intervals.of("2014/P1D"), segments.get(0).getInterval());
+      Assert.assertEquals(NumberedShardSpec.class, segments.get(0).getShardSpec().getClass());
+      Assert.assertEquals(0, segments.get(0).getShardSpec().getPartitionNum());
+
+      verifySchemaAndAggFactory(
+          segmentWithSchemas,
+          RowSignature.builder()
+                      .add("__time", ColumnType.LONG)
+                      .add("ts", ColumnType.STRING)
+                      .add("dim", ColumnType.STRING)
+                      .add("dim_array", ColumnType.STRING)
+                      .add("dim_num_array", ColumnType.STRING)
+                      .add("dimt", ColumnType.STRING)
+                      .add("dimtarray1", ColumnType.STRING)
+                      .add("dimtarray2", ColumnType.STRING)
+                      .add("dimtnum_array", ColumnType.STRING)
+                      .add("val", ColumnType.LONG)
+                      .build(),
+          Collections.singletonMap("val", new LongSumAggregatorFactory("val", "val"))
+      );
+    }
   }
 
   @Test
@@ -776,31 +770,26 @@ public class IndexTaskTest extends IngestionTestBase
           segment.getInterval()
       );
 
-      final Sequence<Cursor> cursorSequence = adapter.getAdapter().makeCursors(
-          null,
-          segment.getInterval(),
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
-      final List<Integer> hashes = cursorSequence
-          .map(cursor -> {
-            final DimensionSelector selector = cursor.getColumnSelectorFactory()
-                                                     .makeDimensionSelector(new DefaultDimensionSpec("dim", "dim"));
-            final int hash = HashPartitionFunction.MURMUR3_32_ABS.hash(
-                HashBasedNumberedShardSpec.serializeGroupKey(
-                    jsonMapper,
-                    Collections.singletonList(selector.getObject())
-                ),
-                hashBasedNumberedShardSpec.getNumBuckets()
-            );
-            cursor.advance();
-            return hash;
-          })
-          .toList();
+      try (final CursorHolder cursorHolder = adapter.getAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        final Cursor cursor = cursorHolder.asCursor();
+        final List<Integer> hashes = new ArrayList<>();
+        final DimensionSelector selector = cursor.getColumnSelectorFactory()
+                                                 .makeDimensionSelector(new DefaultDimensionSpec("dim", "dim"));
+        while (!cursor.isDone()) {
+          final int hash = HashPartitionFunction.MURMUR3_32_ABS.hash(
+              HashBasedNumberedShardSpec.serializeGroupKey(
+                  jsonMapper,
+                  // list of list because partitioning extractKeys uses InputRow.getDimension which always returns a List<String>
+                  Collections.singletonList(Collections.singletonList(selector.getObject()))
+              ),
+              hashBasedNumberedShardSpec.getNumBuckets()
+          );
+          hashes.add(hash);
+          cursor.advance();
+        }
 
-      Assert.assertTrue(hashes.stream().allMatch(h -> h.intValue() == hashes.get(0)));
+        Assert.assertTrue(hashes.stream().allMatch(h -> h.intValue() == hashes.get(0)));
+      }
     }
   }
 
diff --git a/processing/src/main/java/org/apache/druid/frame/field/RowReader.java b/processing/src/main/java/org/apache/druid/frame/field/RowReader.java
index c79c9b25d67..861609ad188 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/RowReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/RowReader.java
@@ -57,7 +57,7 @@ public class RowReader
   /**
    * Read a particular field value as an object.
    *
-   * For performance reasons, prefer {@link org.apache.druid.frame.read.FrameReader#makeCursorFactory}
+   * For performance reasons, prefer {@link org.apache.druid.frame.read.FrameReader#makeCursorHolderFactory}
    * for reading many rows out of a frame.
    */
   public Object readField(final Memory memory, final long rowPosition, final long rowLength, final int fieldNumber)
@@ -77,7 +77,7 @@ public class RowReader
   /**
    * Read an entire row as a list of objects.
    *
-   * For performance reasons, prefer {@link org.apache.druid.frame.read.FrameReader#makeCursorFactory}
+   * For performance reasons, prefer {@link org.apache.druid.frame.read.FrameReader#makeCursorHolderFactory}
    * for reading many rows out of a frame.
    */
   public List<Object> readRow(final Memory memory, final long rowPosition, final long rowLength)
diff --git a/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessors.java b/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessors.java
index 2e21bbc312b..f75ced24b12 100644
--- a/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessors.java
+++ b/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessors.java
@@ -32,9 +32,8 @@ import org.apache.druid.frame.segment.FrameCursor;
 import org.apache.druid.frame.segment.FrameStorageAdapter;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Yielders;
 import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.VirtualColumns;
 
 import java.io.Closeable;
@@ -115,13 +114,16 @@ public class FrameProcessors
       final VirtualColumns virtualColumns
   )
   {
-    // Safe to never close the Sequence that the FrameCursor comes from, because it does not need to be closed.
-    // Refer to FrameStorageAdapter#makeCursors.
-
-    return (FrameCursor) Yielders.each(
-        new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY)
-            .makeCursors(null, Intervals.ETERNITY, virtualColumns, Granularities.ALL, false, null)
-    ).get();
+    final CursorBuildSpec cursorBuildSpec = CursorBuildSpec.builder()
+                                                           .setVirtualColumns(virtualColumns)
+                                                           .build();
+    // Despite appearances of columnar FrameCursorHolderFactory with its closers, it is currently safe to never close
+    // the CursorHolder that the FrameCursor comes from because it really does nothing. The row based
+    // FrameCursorHolderFactory has no closer stuff at all and is totally safe. If this ever changes, this method will
+    // probably need to wrap the cursor in something closeable, or be reworked to just return the CursorHolder so that
+    // callers can deal with closing the stuff.
+    return (FrameCursor) new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY).makeCursorHolder(cursorBuildSpec)
+                                                                                        .asCursor();
   }
 
   /**
diff --git a/processing/src/main/java/org/apache/druid/frame/read/FrameReader.java b/processing/src/main/java/org/apache/druid/frame/read/FrameReader.java
index 46a848fb6b1..ce169b72b1d 100644
--- a/processing/src/main/java/org/apache/druid/frame/read/FrameReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/read/FrameReader.java
@@ -29,10 +29,11 @@ import org.apache.druid.frame.key.FrameComparisonWidgetImpl;
 import org.apache.druid.frame.key.KeyColumn;
 import org.apache.druid.frame.read.columnar.FrameColumnReader;
 import org.apache.druid.frame.read.columnar.FrameColumnReaders;
-import org.apache.druid.frame.segment.row.FrameCursorFactory;
+import org.apache.druid.frame.segment.columnar.ColumnarFrameCursorHolderFactory;
+import org.apache.druid.frame.segment.row.RowFrameCursorHolderFactory;
 import org.apache.druid.frame.write.FrameWriterUtils;
 import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.segment.CursorFactory;
+import org.apache.druid.segment.CursorHolderFactory;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
@@ -136,15 +137,15 @@ public class FrameReader
   }
 
   /**
-   * Create a {@link CursorFactory} for the given frame.
+   * Create a {@link CursorHolderFactory} for the given frame.
    */
-  public CursorFactory makeCursorFactory(final Frame frame)
+  public CursorHolderFactory makeCursorHolderFactory(final Frame frame)
   {
     switch (frame.type()) {
       case COLUMNAR:
-        return new org.apache.druid.frame.segment.columnar.FrameCursorFactory(frame, signature, columnReaders);
+        return new ColumnarFrameCursorHolderFactory(frame, signature, columnReaders);
       case ROW_BASED:
-        return new FrameCursorFactory(frame, this, fieldReaders);
+        return new RowFrameCursorHolderFactory(frame, this, fieldReaders);
       default:
         throw DruidException.defensive("Unrecognized frame type [%s]", frame.type());
     }
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/FrameCursor.java b/processing/src/main/java/org/apache/druid/frame/segment/FrameCursor.java
index 3d4160a7abb..82cb092779d 100644
--- a/processing/src/main/java/org/apache/druid/frame/segment/FrameCursor.java
+++ b/processing/src/main/java/org/apache/druid/frame/segment/FrameCursor.java
@@ -19,16 +19,16 @@
 
 package org.apache.druid.frame.segment;
 
-import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.frame.segment.columnar.ColumnarFrameCursorHolderFactory;
+import org.apache.druid.frame.segment.row.RowFrameCursorHolderFactory;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.SimpleSettableOffset;
-import org.joda.time.DateTime;
 
 /**
- * An implementation of {@link Cursor} used by {@link org.apache.druid.frame.segment.row.FrameCursorFactory}
- * and {@link org.apache.druid.frame.segment.columnar.FrameCursorFactory}.
+ * An implementation of {@link Cursor} used by {@link RowFrameCursorHolderFactory}
+ * and {@link ColumnarFrameCursorHolderFactory}.
  *
  * Adds the methods {@link #getCurrentRow()} and {@link #setCurrentRow(int)} so the cursor can be moved to
  * particular rows.
@@ -53,12 +53,6 @@ public class FrameCursor implements Cursor
     return columnSelectorFactory;
   }
 
-  @Override
-  public DateTime getTime()
-  {
-    return DateTimes.MIN;
-  }
-
   @Override
   public void advance()
   {
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/FrameStorageAdapter.java b/processing/src/main/java/org/apache/druid/frame/segment/FrameStorageAdapter.java
index 9733c548d42..79dcdb7ced3 100644
--- a/processing/src/main/java/org/apache/druid/frame/segment/FrameStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/frame/segment/FrameStorageAdapter.java
@@ -21,21 +21,16 @@ package org.apache.druid.frame.segment;
 
 import org.apache.druid.frame.Frame;
 import org.apache.druid.frame.read.FrameReader;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
-import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.CursorFactory;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.CursorHolderFactory;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.Metadata;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.ListIndexed;
-import org.apache.druid.segment.vector.VectorCursor;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -52,14 +47,14 @@ public class FrameStorageAdapter implements StorageAdapter
   private final Frame frame;
   private final FrameReader frameReader;
   private final Interval interval;
-  private final CursorFactory cursorFactory;
+  private final CursorHolderFactory cursorFactory;
 
   public FrameStorageAdapter(Frame frame, FrameReader frameReader, Interval interval)
   {
     this.frame = frame;
     this.frameReader = frameReader;
     this.interval = interval;
-    this.cursorFactory = frameReader.makeCursorFactory(frame);
+    this.cursorFactory = frameReader.makeCursorHolderFactory(frame);
   }
 
   @Override
@@ -147,49 +142,8 @@ public class FrameStorageAdapter implements StorageAdapter
   }
 
   @Override
-  public boolean canVectorize(@Nullable Filter filter, VirtualColumns virtualColumns, boolean descending)
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    return cursorFactory.canVectorize(filter, virtualColumns, descending);
-  }
-
-  @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      Granularity gran,
-      boolean descending,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
-  {
-    return cursorFactory.makeCursors(
-        filter,
-        interval,
-        virtualColumns,
-        gran,
-        descending,
-        queryMetrics
-    );
-  }
-
-  @Nullable
-  @Override
-  public VectorCursor makeVectorCursor(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      boolean descending,
-      int vectorSize,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
-  {
-    return cursorFactory.makeVectorCursor(
-        filter,
-        interval,
-        virtualColumns,
-        descending,
-        vectorSize,
-        queryMetrics
-    );
+    return cursorFactory.makeCursorHolder(spec);
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorHolderFactory.java b/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorHolderFactory.java
new file mode 100644
index 00000000000..7c97530e37f
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorHolderFactory.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.segment.columnar;
+
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.FrameType;
+import org.apache.druid.frame.read.columnar.FrameColumnReader;
+import org.apache.druid.frame.segment.FrameCursor;
+import org.apache.druid.frame.segment.FrameCursorUtils;
+import org.apache.druid.frame.segment.FrameFilteredOffset;
+import org.apache.druid.frame.segment.row.RowFrameCursorHolderFactory;
+import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.query.filter.Filter;
+import org.apache.druid.query.filter.vector.VectorValueMatcher;
+import org.apache.druid.segment.ColumnCache;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.CursorHolderFactory;
+import org.apache.druid.segment.Cursors;
+import org.apache.druid.segment.QueryableIndexColumnSelectorFactory;
+import org.apache.druid.segment.SimpleAscendingOffset;
+import org.apache.druid.segment.SimpleDescendingOffset;
+import org.apache.druid.segment.SimpleSettableOffset;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.vector.FilteredVectorOffset;
+import org.apache.druid.segment.vector.NoFilterVectorOffset;
+import org.apache.druid.segment.vector.QueryableIndexVectorColumnSelectorFactory;
+import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
+import org.apache.druid.segment.vector.VectorCursor;
+import org.apache.druid.segment.vector.VectorOffset;
+import org.apache.druid.utils.CloseableUtils;
+
+import javax.annotation.Nullable;
+import java.util.List;
+
+/**
+ * A {@link CursorHolderFactory} implementation based on a single columnar {@link Frame}.
+ *
+ * This class is only used for columnar frames. It is not used for row-based frames.
+ *
+ * @see RowFrameCursorHolderFactory the row-based version
+ */
+public class ColumnarFrameCursorHolderFactory implements CursorHolderFactory
+{
+  private final Frame frame;
+  private final RowSignature signature;
+  private final List<FrameColumnReader> columnReaders;
+
+  public ColumnarFrameCursorHolderFactory(
+      final Frame frame,
+      final RowSignature signature,
+      final List<FrameColumnReader> columnReaders
+  )
+  {
+    this.frame = FrameType.COLUMNAR.ensureType(frame);
+    this.signature = signature;
+    this.columnReaders = columnReaders;
+  }
+
+  @Override
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
+  {
+    final Closer closer = Closer.create();
+    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    final List<OrderBy> ordering;
+    final boolean descending;
+    if (Cursors.preferDescendingTimeOrdering(spec)) {
+      ordering = Cursors.descendingTimeOrder();
+      descending = true;
+    } else {
+      ordering = Cursors.ascendingTimeOrder();
+      descending = false;
+    }
+    return new CursorHolder()
+    {
+      @Override
+      public boolean canVectorize()
+      {
+        return (spec.getFilter() == null || spec.getFilter().canVectorizeMatcher(signature))
+               && spec.getVirtualColumns().canVectorize(signature)
+               && !descending;
+      }
+
+      @Override
+      public Cursor asCursor()
+      {
+        final FrameQueryableIndex index = new FrameQueryableIndex(frame, signature, columnReaders);
+        final ColumnCache columnCache = new ColumnCache(index, closer);
+        final Filter filterToUse = FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
+        final boolean descendingTimeOrder = Cursors.preferDescendingTimeOrdering(spec);
+        final SimpleSettableOffset baseOffset = descendingTimeOrder
+                                                ? new SimpleDescendingOffset(frame.numRows())
+                                                : new SimpleAscendingOffset(frame.numRows());
+
+        final QueryableIndexColumnSelectorFactory columnSelectorFactory = new QueryableIndexColumnSelectorFactory(
+                spec.getVirtualColumns(),
+                descendingTimeOrder,
+                baseOffset,
+                columnCache
+        );
+
+        final SimpleSettableOffset offset;
+        if (filterToUse == null) {
+          offset = baseOffset;
+        } else {
+          offset = new FrameFilteredOffset(baseOffset, columnSelectorFactory, filterToUse);
+        }
+
+        return new FrameCursor(offset, columnSelectorFactory);
+      }
+
+      @Override
+      public List<OrderBy> getOrdering()
+      {
+        return ordering;
+      }
+
+      @Nullable
+      @Override
+      public VectorCursor asVectorCursor()
+      {
+        if (!canVectorize()) {
+          throw new ISE("Cannot vectorize. Check 'canVectorize' before calling 'asVectorCursor'.");
+        }
+
+        final FrameQueryableIndex index = new FrameQueryableIndex(frame, signature, columnReaders);
+        final Filter filterToUse = FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
+        final VectorOffset baseOffset = new NoFilterVectorOffset(
+            spec.getQueryContext().getVectorSize(),
+            0,
+            frame.numRows()
+        );
+        final ColumnCache columnCache = new ColumnCache(index, closer);
+
+        // baseSelectorFactory using baseOffset is the column selector for filtering.
+        final VectorColumnSelectorFactory baseSelectorFactory = new QueryableIndexVectorColumnSelectorFactory(
+            index,
+            baseOffset,
+            columnCache,
+            spec.getVirtualColumns()
+        );
+
+        if (filterToUse == null) {
+          return new FrameVectorCursor(baseOffset, baseSelectorFactory);
+        } else {
+          final VectorValueMatcher matcher = filterToUse.makeVectorMatcher(baseSelectorFactory);
+          final FilteredVectorOffset filteredOffset = FilteredVectorOffset.create(
+              baseOffset,
+              matcher
+          );
+
+          final VectorColumnSelectorFactory filteredSelectorFactory = new QueryableIndexVectorColumnSelectorFactory(
+              index,
+              filteredOffset,
+              columnCache,
+              spec.getVirtualColumns()
+          );
+
+          return new FrameVectorCursor(filteredOffset, filteredSelectorFactory);
+        }
+      }
+
+      @Override
+      public void close()
+      {
+        CloseableUtils.closeAndWrapExceptions(closer);
+      }
+    };
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameCursorFactory.java b/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameCursorFactory.java
deleted file mode 100644
index 3497ed39898..00000000000
--- a/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameCursorFactory.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.frame.segment.columnar;
-
-import org.apache.druid.frame.Frame;
-import org.apache.druid.frame.FrameType;
-import org.apache.druid.frame.read.columnar.FrameColumnReader;
-import org.apache.druid.frame.segment.FrameCursor;
-import org.apache.druid.frame.segment.FrameCursorUtils;
-import org.apache.druid.frame.segment.FrameFilteredOffset;
-import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.UOE;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
-import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
-import org.apache.druid.query.filter.vector.VectorValueMatcher;
-import org.apache.druid.segment.ColumnCache;
-import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.CursorFactory;
-import org.apache.druid.segment.QueryableIndexColumnSelectorFactory;
-import org.apache.druid.segment.SimpleAscendingOffset;
-import org.apache.druid.segment.SimpleDescendingOffset;
-import org.apache.druid.segment.SimpleSettableOffset;
-import org.apache.druid.segment.VirtualColumns;
-import org.apache.druid.segment.column.RowSignature;
-import org.apache.druid.segment.vector.FilteredVectorOffset;
-import org.apache.druid.segment.vector.NoFilterVectorOffset;
-import org.apache.druid.segment.vector.QueryableIndexVectorColumnSelectorFactory;
-import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
-import org.apache.druid.segment.vector.VectorCursor;
-import org.apache.druid.segment.vector.VectorOffset;
-import org.joda.time.Interval;
-
-import javax.annotation.Nullable;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * A {@link CursorFactory} implementation based on a single columnar {@link Frame}.
- *
- * This class is only used for columnar frames. It is not used for row-based frames.
- *
- * @see org.apache.druid.frame.segment.row.FrameCursorFactory the row-based version
- */
-public class FrameCursorFactory implements CursorFactory
-{
-  private final Frame frame;
-  private final RowSignature signature;
-  private final List<FrameColumnReader> columnReaders;
-
-  public FrameCursorFactory(
-      final Frame frame,
-      final RowSignature signature,
-      final List<FrameColumnReader> columnReaders
-  )
-  {
-    this.frame = FrameType.COLUMNAR.ensureType(frame);
-    this.signature = signature;
-    this.columnReaders = columnReaders;
-  }
-
-  @Override
-  public boolean canVectorize(@Nullable Filter filter, VirtualColumns virtualColumns, boolean descending)
-  {
-    return (filter == null || filter.canVectorizeMatcher(signature))
-           && virtualColumns.canVectorize(signature)
-           && !descending;
-  }
-
-  @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      Granularity gran,
-      boolean descending,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
-  {
-    final FrameQueryableIndex index = new FrameQueryableIndex(frame, signature, columnReaders);
-
-    if (Granularities.ALL.equals(gran)) {
-      final Closer closer = Closer.create();
-      final Cursor cursor = makeGranularityAllCursor(
-          new ColumnCache(index, closer),
-          frame.numRows(),
-          filter,
-          interval,
-          virtualColumns,
-          descending
-      );
-
-      return Sequences.simple(Collections.singletonList(cursor)).withBaggage(closer);
-    } else {
-      // Not currently needed for the intended use cases of frame-based cursors.
-      throw new UOE("Granularity [%s] not supported", gran);
-    }
-  }
-
-  @Nullable
-  @Override
-  public VectorCursor makeVectorCursor(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      boolean descending,
-      int vectorSize,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
-  {
-    if (!canVectorize(filter, virtualColumns, descending)) {
-      throw new ISE("Cannot vectorize. Check 'canVectorize' before calling 'makeVectorCursor'.");
-    }
-
-    final Closer closer = Closer.create();
-    final FrameQueryableIndex index = new FrameQueryableIndex(frame, signature, columnReaders);
-    final Filter filterToUse = FrameCursorUtils.buildFilter(filter, interval);
-    final VectorOffset baseOffset = new NoFilterVectorOffset(vectorSize, 0, frame.numRows());
-    final ColumnCache columnCache = new ColumnCache(index, closer);
-
-    // baseColumnSelectorFactory using baseOffset is the column selector for filtering.
-    final VectorColumnSelectorFactory baseColumnSelectorFactory = new QueryableIndexVectorColumnSelectorFactory(
-        index,
-        baseOffset,
-        columnCache,
-        virtualColumns
-    );
-
-    if (filterToUse == null) {
-      return new FrameVectorCursor(baseOffset, baseColumnSelectorFactory, closer);
-    } else {
-      final VectorValueMatcher matcher = filterToUse.makeVectorMatcher(baseColumnSelectorFactory);
-      final FilteredVectorOffset filteredOffset = FilteredVectorOffset.create(
-          baseOffset,
-          matcher
-      );
-
-      final VectorColumnSelectorFactory filteredColumnSelectorFactory = new QueryableIndexVectorColumnSelectorFactory(
-          index,
-          filteredOffset,
-          columnCache,
-          virtualColumns
-      );
-
-      return new FrameVectorCursor(filteredOffset, filteredColumnSelectorFactory, closer);
-    }
-  }
-
-  private static Cursor makeGranularityAllCursor(
-      final ColumnCache columnSelector,
-      final int numRows,
-      @Nullable final Filter filter,
-      final Interval interval,
-      final VirtualColumns virtualColumns,
-      final boolean descending
-  )
-  {
-    final Filter filterToUse = FrameCursorUtils.buildFilter(filter, interval);
-    final SimpleSettableOffset baseOffset =
-        descending ? new SimpleDescendingOffset(numRows) : new SimpleAscendingOffset(numRows);
-    final SimpleSettableOffset offset;
-
-    final QueryableIndexColumnSelectorFactory columnSelectorFactory =
-        new QueryableIndexColumnSelectorFactory(virtualColumns, descending, baseOffset, columnSelector);
-
-    if (filterToUse == null) {
-      offset = baseOffset;
-    } else {
-      offset = new FrameFilteredOffset(baseOffset, columnSelectorFactory, filterToUse);
-    }
-
-    return new FrameCursor(offset, columnSelectorFactory);
-  }
-}
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameQueryableIndex.java b/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameQueryableIndex.java
index f42a33ce6f7..86bf7eb5126 100644
--- a/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameQueryableIndex.java
+++ b/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameQueryableIndex.java
@@ -42,7 +42,7 @@ import java.util.Map;
  * A {@link QueryableIndex} implementation based on a single columnar {@link Frame}. There is no internal caching
  * of columns here, so callers should generally wrap this in a {@link org.apache.druid.segment.ColumnCache}.
  *
- * This class exists so {@link FrameCursorFactory} can reuse code meant for regular segment-backed
+ * This class exists so {@link ColumnarFrameCursorHolderFactory} can reuse code meant for regular segment-backed
  * {@link QueryableIndex}. Some methods are implemented by throwing {@link UnsupportedOperationException}, wherever
  * it is not expected that those methods are actually going to be needed.
  */
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameVectorCursor.java b/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameVectorCursor.java
index da1cbf9b0ce..e7789ed0375 100644
--- a/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameVectorCursor.java
+++ b/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameVectorCursor.java
@@ -20,14 +20,11 @@
 package org.apache.druid.frame.segment.columnar;
 
 import org.apache.druid.frame.Frame;
-import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
 import org.apache.druid.segment.vector.VectorCursor;
 import org.apache.druid.segment.vector.VectorOffset;
 
-import java.io.IOException;
-
 /**
  * A {@link VectorCursor} that is based on a {@link Frame}.
  *
@@ -37,17 +34,14 @@ public class FrameVectorCursor implements VectorCursor
 {
   private final VectorOffset offset;
   private final VectorColumnSelectorFactory columnSelectorFactory;
-  private final Closer closer;
 
   FrameVectorCursor(
       final VectorOffset offset,
-      final VectorColumnSelectorFactory columnSelectorFactory,
-      final Closer closer
+      final VectorColumnSelectorFactory columnSelectorFactory
   )
   {
     this.offset = offset;
     this.columnSelectorFactory = columnSelectorFactory;
-    this.closer = closer;
   }
 
   @Override
@@ -75,17 +69,6 @@ public class FrameVectorCursor implements VectorCursor
     offset.reset();
   }
 
-  @Override
-  public void close()
-  {
-    try {
-      closer.close();
-    }
-    catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
   @Override
   public int getMaxVectorSize()
   {
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/row/FrameCursorFactory.java b/processing/src/main/java/org/apache/druid/frame/segment/row/FrameCursorFactory.java
deleted file mode 100644
index b3232e15d7f..00000000000
--- a/processing/src/main/java/org/apache/druid/frame/segment/row/FrameCursorFactory.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.frame.segment.row;
-
-import org.apache.druid.frame.Frame;
-import org.apache.druid.frame.FrameType;
-import org.apache.druid.frame.field.FieldReader;
-import org.apache.druid.frame.read.FrameReader;
-import org.apache.druid.frame.segment.FrameCursor;
-import org.apache.druid.frame.segment.FrameCursorUtils;
-import org.apache.druid.frame.segment.FrameFilteredOffset;
-import org.apache.druid.java.util.common.UOE;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
-import org.apache.druid.segment.ColumnSelectorFactory;
-import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.CursorFactory;
-import org.apache.druid.segment.SimpleAscendingOffset;
-import org.apache.druid.segment.SimpleDescendingOffset;
-import org.apache.druid.segment.SimpleSettableOffset;
-import org.apache.druid.segment.VirtualColumns;
-import org.joda.time.Interval;
-
-import javax.annotation.Nullable;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * A {@link CursorFactory} implementation based on a single row-based {@link Frame}.
- *
- * This class is only used for row-based frames.
- *
- * @see org.apache.druid.frame.segment.columnar.FrameCursorFactory the columnar version
- */
-public class FrameCursorFactory implements CursorFactory
-{
-  private final Frame frame;
-  private final FrameReader frameReader;
-  private final List<FieldReader> fieldReaders;
-
-  public FrameCursorFactory(
-      final Frame frame,
-      final FrameReader frameReader,
-      final List<FieldReader> fieldReaders
-  )
-  {
-    this.frame = FrameType.ROW_BASED.ensureType(frame);
-    this.frameReader = frameReader;
-    this.fieldReaders = fieldReaders;
-  }
-
-  @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      Granularity gran,
-      boolean descending,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
-  {
-    if (Granularities.ALL.equals(gran)) {
-      final Cursor cursor = makeGranularityAllCursor(filter, interval, virtualColumns, descending);
-
-      // Note: if anything closeable is ever added to this Sequence, make sure to update FrameProcessors.makeCursor.
-      // Currently, it assumes that closing the Sequence does nothing.
-      return Sequences.simple(Collections.singletonList(cursor));
-    } else {
-      // Not currently needed for the intended use cases of frame-based cursors.
-      throw new UOE("Granularity [%s] not supported", gran);
-    }
-  }
-
-  private Cursor makeGranularityAllCursor(
-      @Nullable final Filter filter,
-      final Interval interval,
-      final VirtualColumns virtualColumns,
-      final boolean descending
-  )
-  {
-    final Filter filterToUse = FrameCursorUtils.buildFilter(filter, interval);
-
-    final SimpleSettableOffset baseOffset = descending
-                              ? new SimpleDescendingOffset(frame.numRows())
-                              : new SimpleAscendingOffset(frame.numRows());
-
-    final SimpleSettableOffset offset;
-
-    final ColumnSelectorFactory columnSelectorFactory =
-        virtualColumns.wrap(
-            new FrameColumnSelectorFactory(
-                frame,
-                frameReader.signature(),
-                fieldReaders,
-                new CursorFrameRowPointer(frame, baseOffset)
-            )
-        );
-
-    if (filterToUse == null) {
-      offset = baseOffset;
-    } else {
-      offset = new FrameFilteredOffset(baseOffset, columnSelectorFactory, filterToUse);
-    }
-
-    return new FrameCursor(offset, columnSelectorFactory);
-  }
-}
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorHolderFactory.java b/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorHolderFactory.java
new file mode 100644
index 00000000000..2d0d6567d5f
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorHolderFactory.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.segment.row;
+
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.FrameType;
+import org.apache.druid.frame.field.FieldReader;
+import org.apache.druid.frame.read.FrameReader;
+import org.apache.druid.frame.segment.FrameCursor;
+import org.apache.druid.frame.segment.FrameCursorUtils;
+import org.apache.druid.frame.segment.FrameFilteredOffset;
+import org.apache.druid.frame.segment.columnar.ColumnarFrameCursorHolderFactory;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.query.filter.Filter;
+import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.CursorHolderFactory;
+import org.apache.druid.segment.Cursors;
+import org.apache.druid.segment.SimpleAscendingOffset;
+import org.apache.druid.segment.SimpleDescendingOffset;
+import org.apache.druid.segment.SimpleSettableOffset;
+
+import javax.annotation.Nullable;
+import java.util.List;
+
+/**
+ * A {@link CursorHolderFactory} implementation based on a single row-based {@link Frame}.
+ *
+ * This class is only used for row-based frames.
+ *
+ * @see ColumnarFrameCursorHolderFactory the columnar version
+ */
+public class RowFrameCursorHolderFactory implements CursorHolderFactory
+{
+  private final Frame frame;
+  private final FrameReader frameReader;
+  private final List<FieldReader> fieldReaders;
+
+  public RowFrameCursorHolderFactory(
+      final Frame frame,
+      final FrameReader frameReader,
+      final List<FieldReader> fieldReaders
+  )
+  {
+    this.frame = FrameType.ROW_BASED.ensureType(frame);
+    this.frameReader = frameReader;
+    this.fieldReaders = fieldReaders;
+  }
+
+  @Override
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
+  {
+    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    final List<OrderBy> ordering;
+    final boolean descending;
+    if (Cursors.preferDescendingTimeOrdering(spec)) {
+      ordering = Cursors.descendingTimeOrder();
+      descending = true;
+    } else {
+      ordering = Cursors.ascendingTimeOrder();
+      descending = false;
+    }
+    return new CursorHolder()
+    {
+      @Nullable
+      @Override
+      public Cursor asCursor()
+      {
+        final Filter filterToUse = FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
+
+        final SimpleSettableOffset baseOffset = descending
+                                                ? new SimpleDescendingOffset(frame.numRows())
+                                                : new SimpleAscendingOffset(frame.numRows());
+
+
+        final ColumnSelectorFactory columnSelectorFactory =
+            spec.getVirtualColumns().wrap(
+                new FrameColumnSelectorFactory(
+                    frame,
+                    frameReader.signature(),
+                    fieldReaders,
+                    new CursorFrameRowPointer(frame, baseOffset)
+                )
+            );
+
+        final SimpleSettableOffset offset;
+        if (filterToUse == null) {
+          offset = baseOffset;
+        } else {
+          offset = new FrameFilteredOffset(baseOffset, columnSelectorFactory, filterToUse);
+        }
+
+        final FrameCursor cursor = new FrameCursor(offset, columnSelectorFactory);
+
+        // Note: if anything closeable is ever added to this Sequence, make sure to update FrameProcessors.makeCursor.
+        // Currently, it assumes that closing the Sequence does nothing.
+        return cursor;
+      }
+
+      @Nullable
+      @Override
+      public List<OrderBy> getOrdering()
+      {
+        return ordering;
+      }
+    };
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java b/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
index 303f835d889..3c8c89b2c26 100644
--- a/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
+++ b/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
@@ -19,6 +19,26 @@
 
 package org.apache.druid.java.util.common.granularity;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.query.Query;
+import org.apache.druid.query.expression.TimestampFloorExprMacro;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.VirtualColumn;
+import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
+import org.joda.time.chrono.ISOChronology;
+
+import javax.annotation.Nullable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+
 /**
  * This class was created b/c sometimes static initializers of a class that use a subclass can deadlock.
  * See: #2979, #3979
@@ -42,8 +62,103 @@ public class Granularities
   public static final Granularity ALL = GranularityType.ALL.getDefaultGranularity();
   public static final Granularity NONE = GranularityType.NONE.getDefaultGranularity();
 
+  public static final String GRANULARITY_VIRTUAL_COLUMN_NAME = "__virtualGranularity";
+
   public static Granularity nullToAll(Granularity granularity)
   {
     return granularity == null ? Granularities.ALL : granularity;
   }
+
+
+  /**
+   * Decorates {@link CursorBuildSpec} with a grouping column and virtual column equivalent to the {@link Granularity}
+   * for a {@link Query}, if that query has granularity other than {@link Granularities#ALL}. If the query has 'ALL'
+   * granularity, the {@link CursorBuildSpec} will be returned as is.
+   * The reason for this is that a query with a granularity that is not 'ALL' is effectively an additional grouping on
+   * TIME_FLOOR(__time, granularity), so by adding this to the {@link CursorBuildSpec} allows a
+   * {@link org.apache.druid.segment.CursorHolder} to potentially specialize any {@link org.apache.druid.segment.Cursor}
+   * or {@link org.apache.druid.segment.vector.VectorCursor} it provides by using pre-aggregated data at the closest
+   * matching granularity if available.
+   *
+   * @see #toVirtualColumn(Granularity, String) for how {@link Granularity} is translated into a {@link VirtualColumn}
+   */
+  public static CursorBuildSpec decorateCursorBuildSpec(Query<?> query, CursorBuildSpec buildSpec)
+  {
+    // ALL granularity
+    if (ALL.equals(query.getGranularity())) {
+      return buildSpec;
+    }
+    // For any other granularity, we add a grouping on TIME_FLOOR(__time, granularity) or __time itself
+    final ImmutableSet.Builder<String> builder = ImmutableSet.builder();
+    final Set<String> requiredColumns = query.getRequiredColumns();
+    if (requiredColumns != null) {
+      builder.addAll(requiredColumns);
+    }
+    builder.addAll(query.getVirtualColumns().getColumnNames());
+    final Set<String> columnNamesForConflictResolution = builder.build();
+    String virtualColumnName = GRANULARITY_VIRTUAL_COLUMN_NAME;
+    int ctr = 0;
+    while (columnNamesForConflictResolution.contains(virtualColumnName)) {
+      virtualColumnName = virtualColumnName + ctr++;
+    }
+
+    final VirtualColumn granularityVirtual = toVirtualColumn(query.getGranularity(), virtualColumnName);
+    // granularityVirtual will not be null because we have already filtered out ALL granularity, but check just in case
+    Preconditions.checkNotNull(
+        granularityVirtual,
+        "Granularity virtual column is null for granularity[%s]",
+        query.getGranularity()
+    );
+
+    final VirtualColumns virtualColumns = VirtualColumns.fromIterable(
+        Iterables.concat(
+            Collections.singletonList(granularityVirtual),
+            () -> Arrays.stream(buildSpec.getVirtualColumns().getVirtualColumns()).iterator()
+        )
+    );
+    final ImmutableList.Builder<String> groupingColumnsBuilder = ImmutableList.builder();
+    groupingColumnsBuilder.add(granularityVirtual.getOutputName());
+    if (buildSpec.getGroupingColumns() != null) {
+      groupingColumnsBuilder.addAll(buildSpec.getGroupingColumns());
+    }
+    return CursorBuildSpec.builder(buildSpec)
+                          .setVirtualColumns(virtualColumns)
+                          .setGroupingColumns(groupingColumnsBuilder.build())
+                          .build();
+  }
+
+  /**
+   * Translates a {@link Granularity} to a {@link ExpressionVirtualColumn} on {@link ColumnHolder#TIME_COLUMN_NAME} of
+   * the equivalent grouping column. If granularity is {@link #ALL}, this method returns null since we are not grouping
+   * on time. If granularity is a {@link PeriodGranularity} with UTC timezone and no origin, this method returns a
+   * virtual column with {@link TimestampFloorExprMacro.TimestampFloorExpr} of the specified period. If granularity is
+   * {@link #NONE}, or any other kind of granularity (duration, period with non-utc timezone or origin) this method
+   * returns a virtual column with {@link org.apache.druid.math.expr.IdentifierExpr} specifying
+   * {@link ColumnHolder#TIME_COLUMN_NAME} directly.
+   */
+  @Nullable
+  public static ExpressionVirtualColumn toVirtualColumn(Granularity granularity, String virtualColumnName)
+  {
+    if (ALL.equals(granularity)) {
+      return null;
+    }
+    final String expression;
+    if (NONE.equals(granularity) || granularity instanceof DurationGranularity) {
+      expression = ColumnHolder.TIME_COLUMN_NAME;
+    } else {
+      PeriodGranularity period = (PeriodGranularity) granularity;
+      if (!ISOChronology.getInstanceUTC().getZone().equals(period.getTimeZone()) || period.getOrigin() != null) {
+        expression = ColumnHolder.TIME_COLUMN_NAME;
+      } else {
+        expression = TimestampFloorExprMacro.forQueryGranularity(period.getPeriod());
+      }
+    }
+
+    return new ExpressionVirtualColumn(
+        virtualColumnName,
+        expression,
+        ColumnType.LONG,
+        ExprMacroTable.granularity()
+    );
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java b/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java
index a9f5b941c78..699eb8967eb 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java
@@ -27,6 +27,7 @@ import com.google.common.collect.Maps;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.math.expr.vector.ExprVectorProcessor;
 import org.apache.druid.math.expr.vector.FallbackVectorProcessor;
+import org.apache.druid.query.expression.TimestampFloorExprMacro;
 
 import javax.annotation.Nullable;
 import java.util.Collections;
@@ -54,6 +55,10 @@ public class ExprMacroTable
   );
   private static final ExprMacroTable NIL = new ExprMacroTable(Collections.emptyList());
 
+  private static final ExprMacroTable TIME_FLOOR_MACRO_TABLE = new ExprMacroTable(
+      Collections.singletonList(new TimestampFloorExprMacro())
+  );
+
   private final Map<String, ExprMacro> macroMap;
 
   public ExprMacroTable(final List<ExprMacro> macros)
@@ -68,6 +73,16 @@ public class ExprMacroTable
     return NIL;
   }
 
+  /**
+   * Specialized {@link ExprMacroTable} that only knows about {@link TimestampFloorExprMacro}, intended for use
+   * parsing generated expressions which translate {@link org.apache.druid.java.util.common.granularity.Granularity}
+   * into {@link Expr}
+   */
+  public static ExprMacroTable granularity()
+  {
+    return TIME_FLOOR_MACRO_TABLE;
+  }
+
   public List<ExprMacro> getMacros()
   {
     return ImmutableList.copyOf(macroMap.values());
diff --git a/processing/src/main/java/org/apache/druid/query/BaseQuery.java b/processing/src/main/java/org/apache/druid/query/BaseQuery.java
index ed97c549b62..f25a5ef317e 100644
--- a/processing/src/main/java/org/apache/druid/query/BaseQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/BaseQuery.java
@@ -66,11 +66,10 @@ public abstract class BaseQuery<T> implements Query<T>
   public BaseQuery(
       DataSource dataSource,
       QuerySegmentSpec querySegmentSpec,
-      boolean descending,
       Map<String, Object> context
   )
   {
-    this(dataSource, querySegmentSpec, descending, context, Granularities.ALL);
+    this(dataSource, querySegmentSpec, false, context, Granularities.ALL);
   }
 
   public BaseQuery(
diff --git a/processing/src/main/java/org/apache/druid/query/CursorGranularizer.java b/processing/src/main/java/org/apache/druid/query/CursorGranularizer.java
new file mode 100644
index 00000000000..b9d3418779a
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/query/CursorGranularizer.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+
+import javax.annotation.Nullable;
+
+/**
+ * Class that helps non-vectorized query engines handle "granularity" parameters. Given a set of intervals, this class
+ * provides mechansims to advance a cursor to the start of an interval ({@link #advanceToBucket(Interval)}),
+ * advance a cursor within a bucket interval ({@link #advanceCursorWithinBucket()}), and check if the current cursor
+ * position is within the bucket {@link #currentOffsetWithinBucket()}.
+ *
+ * @see org.apache.druid.query.vector.VectorCursorGranularizer for vectorized query engines.
+ */
+public class CursorGranularizer
+{
+  @Nullable
+  public static CursorGranularizer create(
+      final StorageAdapter storageAdapter,
+      final Cursor cursor,
+      final Granularity granularity,
+      final Interval queryInterval,
+      final boolean descending
+  )
+  {
+    final DateTime minTime = storageAdapter.getMinTime();
+    final DateTime maxTime = storageAdapter.getMaxTime();
+
+    final Interval storageAdapterInterval = new Interval(minTime, granularity.bucketEnd(maxTime));
+    final Interval clippedQueryInterval = queryInterval.overlap(storageAdapterInterval);
+
+    if (clippedQueryInterval == null) {
+      return null;
+    }
+
+    Iterable<Interval> bucketIterable = granularity.getIterable(clippedQueryInterval);
+    if (descending) {
+      bucketIterable = Lists.reverse(ImmutableList.copyOf(bucketIterable));
+    }
+    final Interval firstBucket = granularity.bucket(clippedQueryInterval.getStart());
+
+    final ColumnValueSelector timeSelector;
+    if (firstBucket.contains(clippedQueryInterval)) {
+      // Only one bucket, no need to read the time column.
+      assert Iterables.size(bucketIterable) == 1;
+      timeSelector = null;
+    } else {
+      // Multiple buckets, need to read the time column to know when we move from one to the next.
+      timeSelector = cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME);
+    }
+
+    return new CursorGranularizer(cursor, bucketIterable, timeSelector, descending);
+  }
+
+  private final Cursor cursor;
+
+  // Iterable that iterates over time buckets.
+  private final Iterable<Interval> bucketIterable;
+
+  // Selector for the "__time" column.
+  @Nullable
+  private final ColumnValueSelector timeSelector;
+  private final boolean descending;
+
+  private long currentBucketStart;
+  private long currentBucketEnd;
+
+  private CursorGranularizer(
+      Cursor cursor,
+      Iterable<Interval> bucketIterable,
+      @Nullable ColumnValueSelector timeSelector,
+      boolean descending
+  )
+  {
+    this.cursor = cursor;
+    this.bucketIterable = bucketIterable;
+    this.timeSelector = timeSelector;
+    this.descending = descending;
+  }
+
+  public Iterable<Interval> getBucketIterable()
+  {
+    return bucketIterable;
+  }
+
+  public DateTime getBucketStart()
+  {
+    return DateTimes.utc(currentBucketStart);
+  }
+
+  public boolean advanceToBucket(final Interval bucketInterval)
+  {
+    if (cursor.isDone()) {
+      return false;
+    }
+    currentBucketStart = bucketInterval.getStartMillis();
+    currentBucketEnd = bucketInterval.getEndMillis();
+    if (timeSelector == null) {
+      return true;
+    }
+    long currentTime = timeSelector.getLong();
+    if (descending) {
+      while (currentTime >= currentBucketEnd && !cursor.isDone()) {
+        cursor.advance();
+        currentTime = timeSelector.getLong();
+      }
+    } else {
+      while (currentTime < currentBucketStart && !cursor.isDone()) {
+        cursor.advance();
+        currentTime = timeSelector.getLong();
+      }
+    }
+
+    return currentBucketStart <= currentTime && currentTime < currentBucketEnd;
+  }
+
+  public boolean advanceCursorWithinBucket()
+  {
+    if (cursor.isDone()) {
+      return false;
+    }
+    cursor.advance();
+    return currentOffsetWithinBucket();
+  }
+
+  public boolean advanceCursorWithinBucketUninterruptedly()
+  {
+    if (cursor.isDone()) {
+      return false;
+    }
+    cursor.advanceUninterruptibly();
+    return currentOffsetWithinBucket();
+  }
+
+  public boolean currentOffsetWithinBucket()
+  {
+    if (cursor.isDone()) {
+      return false;
+    }
+    if (timeSelector == null) {
+      return true;
+    }
+    final long currentTime = timeSelector.getLong();
+    return currentBucketStart <= currentTime && currentTime < currentBucketEnd;
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/query/Druids.java b/processing/src/main/java/org/apache/druid/query/Druids.java
index 824444410c0..aad70c9ecb5 100644
--- a/processing/src/main/java/org/apache/druid/query/Druids.java
+++ b/processing/src/main/java/org/apache/druid/query/Druids.java
@@ -824,8 +824,8 @@ public class Druids
     private long limit;
     private DimFilter dimFilter;
     private List<String> columns = new ArrayList<>();
-    private ScanQuery.Order order;
-    private List<ScanQuery.OrderBy> orderBy;
+    private Order order;
+    private List<OrderBy> orderBy;
     private List<ColumnType> columnTypes = null;
 
     public ScanQuery build()
@@ -956,13 +956,13 @@ public class Druids
       return this;
     }
 
-    public ScanQueryBuilder order(ScanQuery.Order order)
+    public ScanQueryBuilder order(Order order)
     {
       this.order = order;
       return this;
     }
 
-    public ScanQueryBuilder orderBy(List<ScanQuery.OrderBy> orderBys)
+    public ScanQueryBuilder orderBy(List<OrderBy> orderBys)
     {
       this.orderBy = orderBys;
       return this;
diff --git a/processing/src/main/java/org/apache/druid/query/FilteredDataSource.java b/processing/src/main/java/org/apache/druid/query/FilteredDataSource.java
index d5e7186e954..62b42b15c78 100644
--- a/processing/src/main/java/org/apache/druid/query/FilteredDataSource.java
+++ b/processing/src/main/java/org/apache/druid/query/FilteredDataSource.java
@@ -30,6 +30,7 @@ import org.apache.druid.segment.SegmentReference;
 import org.apache.druid.segment.WrappedSegmentReference;
 import org.apache.druid.utils.JvmUtils;
 
+import javax.annotation.Nullable;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -67,7 +68,7 @@ public class FilteredDataSource implements DataSource
     return filter;
   }
 
-  private FilteredDataSource(DataSource base, DimFilter filter)
+  private FilteredDataSource(DataSource base, @Nullable DimFilter filter)
   {
     this.base = base;
     this.filter = filter;
@@ -76,7 +77,7 @@ public class FilteredDataSource implements DataSource
   @JsonCreator
   public static FilteredDataSource create(
       @JsonProperty("base") DataSource base,
-      @JsonProperty("filter") DimFilter f
+      @JsonProperty("filter") @Nullable DimFilter f
   )
   {
     return new FilteredDataSource(base, f);
diff --git a/processing/src/main/java/org/apache/druid/query/FrameBasedInlineDataSource.java b/processing/src/main/java/org/apache/druid/query/FrameBasedInlineDataSource.java
index a0b7bda150d..38a8b1e91d5 100644
--- a/processing/src/main/java/org/apache/druid/query/FrameBasedInlineDataSource.java
+++ b/processing/src/main/java/org/apache/druid/query/FrameBasedInlineDataSource.java
@@ -24,15 +24,15 @@ import org.apache.druid.frame.read.FrameReader;
 import org.apache.druid.frame.segment.FrameStorageAdapter;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.query.planning.DataSourceAnalysis;
 import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.SegmentReference;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.RowSignature;
 
 import java.util.Collections;
@@ -81,7 +81,6 @@ public class FrameBasedInlineDataSource implements DataSource
 
   public Sequence<Object[]> getRowsAsSequence()
   {
-
     final Sequence<Cursor> cursorSequence =
         Sequences.simple(frames)
                  .flatMap(
@@ -89,8 +88,11 @@ public class FrameBasedInlineDataSource implements DataSource
                        Frame frame = frameSignaturePair.getFrame();
                        RowSignature frameSignature = frameSignaturePair.getRowSignature();
                        FrameReader frameReader = FrameReader.create(frameSignature);
-                       return new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY)
-                           .makeCursors(null, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null);
+                       final CursorHolder holder =
+                           new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY).makeCursorHolder(
+                               CursorBuildSpec.FULL_SCAN
+                           );
+                       return Sequences.simple(Collections.singletonList(holder.asCursor())).withBaggage(holder);
                      }
                  );
 
diff --git a/processing/src/main/java/org/apache/druid/query/IterableRowsCursorHelper.java b/processing/src/main/java/org/apache/druid/query/IterableRowsCursorHelper.java
index 4bf1cb92a61..46a39ad4cb5 100644
--- a/processing/src/main/java/org/apache/druid/query/IterableRowsCursorHelper.java
+++ b/processing/src/main/java/org/apache/druid/query/IterableRowsCursorHelper.java
@@ -21,7 +21,6 @@ package org.apache.druid.query;
 
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.guava.Yielder;
@@ -77,7 +76,6 @@ public class IterableRowsCursorHelper
         null,
         Intervals.ETERNITY, // Setting the interval to eternity ensures that we are iterating over all of the rows
         VirtualColumns.EMPTY,
-        Granularities.ALL,
         false,
         rowSignature != null ? rowSignature : RowSignature.empty()
     );
diff --git a/processing/src/main/java/org/apache/druid/query/Order.java b/processing/src/main/java/org/apache/druid/query/Order.java
new file mode 100644
index 00000000000..464aa9ed7d2
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/query/Order.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.druid.java.util.common.StringUtils;
+
+public enum Order
+{
+  ASCENDING,
+  DESCENDING,
+  NONE;
+
+  @JsonValue
+  @Override
+  public String toString()
+  {
+    return StringUtils.toLowerCase(this.name());
+  }
+
+  @JsonCreator
+  public static Order fromString(String name)
+  {
+    return valueOf(StringUtils.toUpperCase(name));
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/query/OrderBy.java b/processing/src/main/java/org/apache/druid/query/OrderBy.java
new file mode 100644
index 00000000000..5cdda9bb46e
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/query/OrderBy.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.StringUtils;
+
+import java.util.Objects;
+
+public class OrderBy
+{
+  public static OrderBy ascending(String columnName)
+  {
+    return new OrderBy(columnName, Order.ASCENDING);
+  }
+
+  public static OrderBy descending(String columnName)
+  {
+    return new OrderBy(columnName, Order.DESCENDING);
+  }
+
+  private final String columnName;
+  private final Order order;
+
+  @JsonCreator
+  public OrderBy(
+      @JsonProperty("columnName") final String columnName,
+      @JsonProperty("order") final Order order
+  )
+  {
+    this.columnName = Preconditions.checkNotNull(columnName, "columnName");
+    this.order = Preconditions.checkNotNull(order, "order");
+
+    if (order == Order.NONE) {
+      throw new IAE("Order required for column [%s]", columnName);
+    }
+  }
+
+  @JsonProperty
+  public String getColumnName()
+  {
+    return columnName;
+  }
+
+  @JsonProperty
+  public Order getOrder()
+  {
+    return order;
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    OrderBy that = (OrderBy) o;
+    return Objects.equals(columnName, that.columnName) && order == that.order;
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hash(columnName, order);
+  }
+
+  @Override
+  public String toString()
+  {
+    return StringUtils.format("%s %s", columnName, order == Order.ASCENDING ? "ASC" : "DESC");
+  }
+
+}
diff --git a/processing/src/main/java/org/apache/druid/query/Queries.java b/processing/src/main/java/org/apache/druid/query/Queries.java
index c08f63938bb..e3262872d7a 100644
--- a/processing/src/main/java/org/apache/druid/query/Queries.java
+++ b/processing/src/main/java/org/apache/druid/query/Queries.java
@@ -27,7 +27,6 @@ import org.apache.druid.guice.annotations.PublicApi;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.PostAggregator;
-import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.planning.DataSourceAnalysis;
 import org.apache.druid.query.spec.MultipleSpecificSegmentSpec;
@@ -36,7 +35,6 @@ import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
 
 import javax.annotation.Nullable;
-
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -206,17 +204,15 @@ public class Queries
    *
    * @param virtualColumns    virtual columns whose inputs should be included.
    * @param filter            optional filter whose inputs should be included.
-   * @param dimensions        dimension specs whose inputs should be included.
-   * @param aggregators       aggregators whose inputs should be included.
-   * @param additionalColumns additional columns to include. Each of these will be added to the returned set, unless it
+   * @param columns           additional columns to include. Each of these will be added to the returned set, unless it
    *                          refers to a virtual column, in which case the virtual column inputs will be added instead.
+   * @param aggregators       aggregators whose inputs should be included.
    */
   public static Set<String> computeRequiredColumns(
       final VirtualColumns virtualColumns,
       @Nullable final DimFilter filter,
-      final List<DimensionSpec> dimensions,
-      final List<AggregatorFactory> aggregators,
-      final List<String> additionalColumns
+      final List<String> columns,
+      final List<AggregatorFactory> aggregators
   )
   {
     final Set<String> requiredColumns = new HashSet<>();
@@ -240,9 +236,9 @@ public class Queries
       }
     }
 
-    for (DimensionSpec dimensionSpec : dimensions) {
-      if (!virtualColumns.exists(dimensionSpec.getDimension())) {
-        requiredColumns.add(dimensionSpec.getDimension());
+    for (String column : columns) {
+      if (!virtualColumns.exists(column)) {
+        requiredColumns.add(column);
       }
     }
 
@@ -254,12 +250,6 @@ public class Queries
       }
     }
 
-    for (String column : additionalColumns) {
-      if (!virtualColumns.exists(column)) {
-        requiredColumns.add(column);
-      }
-    }
-
     return requiredColumns;
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java
index 90b43469dce..fa1f42d4c13 100644
--- a/processing/src/main/java/org/apache/druid/query/Query.java
+++ b/processing/src/main/java/org/apache/druid/query/Query.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Ordering;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.guice.annotations.ExtensionPoint;
 import org.apache.druid.java.util.common.HumanReadableBytes;
 import org.apache.druid.java.util.common.granularity.Granularity;
@@ -41,6 +42,7 @@ import org.apache.druid.query.timeseries.TimeseriesQuery;
 import org.apache.druid.query.topn.TopNQuery;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.utils.CollectionUtils;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Duration;
 import org.joda.time.Interval;
@@ -271,4 +273,18 @@ public interface Query<T>
   {
     return null;
   }
+
+  /**
+   * Returns an interval if {@link #getIntervals()} has only a single interval, else explodes
+   */
+  default Interval getSingleInterval()
+  {
+    return CollectionUtils.getOnlyElement(
+        getIntervals(),
+        (i) -> DruidException.defensive(
+            "This method can only be called after query is reduced to a single segment interval, got [%s]",
+            i
+        )
+    );
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/QueryContexts.java b/processing/src/main/java/org/apache/druid/query/QueryContexts.java
index 402001efe7d..d3545a8494a 100644
--- a/processing/src/main/java/org/apache/druid/query/QueryContexts.java
+++ b/processing/src/main/java/org/apache/druid/query/QueryContexts.java
@@ -29,7 +29,6 @@ import org.apache.druid.java.util.common.Numbers;
 import org.apache.druid.java.util.common.StringUtils;
 
 import javax.annotation.Nullable;
-
 import java.math.BigDecimal;
 import java.util.Arrays;
 import java.util.HashMap;
diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerHelper.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerHelper.java
index 88b12e81c8b..bb9837bac2d 100644
--- a/processing/src/main/java/org/apache/druid/query/QueryRunnerHelper.java
+++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerHelper.java
@@ -19,50 +19,16 @@
 
 package org.apache.druid.query;
 
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.query.context.ResponseContext;
-import org.apache.druid.query.filter.Filter;
-import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
-import org.joda.time.Interval;
 
-import javax.annotation.Nullable;
 import java.io.Closeable;
-import java.util.List;
-import java.util.Objects;
 
 /**
  */
 public class QueryRunnerHelper
 {
-  public static <T> Sequence<Result<T>> makeCursorBasedQuery(
-      final StorageAdapter adapter,
-      final List<Interval> queryIntervals,
-      final Filter filter,
-      final VirtualColumns virtualColumns,
-      final boolean descending,
-      final Granularity granularity,
-      final Function<Cursor, Result<T>> mapFn,
-      @Nullable final QueryMetrics<?> queryMetrics
-  )
-  {
-    Preconditions.checkArgument(
-        queryIntervals.size() == 1, "Can only handle a single interval, got[%s]", queryIntervals
-    );
-
-    return Sequences.filter(
-        Sequences.map(
-            adapter.makeCursors(filter, queryIntervals.get(0), virtualColumns, granularity, descending, queryMetrics),
-            mapFn
-        ),
-        Objects::nonNull
-    );
-  }
 
   public static <T> QueryRunner<T> makeClosingQueryRunner(final QueryRunner<T> runner, final Closeable closeable)
   {
diff --git a/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java
index 37e6a48ea48..5c43c6445bc 100644
--- a/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java
@@ -51,7 +51,7 @@ public class DataSourceMetadataQuery extends BaseQuery<Result<DataSourceMetadata
       @JsonProperty("context") Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec == null ? DEFAULT_SEGMENT_SPEC : querySegmentSpec, false, context);
+    super(dataSource, querySegmentSpec == null ? DEFAULT_SEGMENT_SPEC : querySegmentSpec, context);
   }
 
   @Override
diff --git a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java
index 02eed7327f1..5cd2c88fc80 100644
--- a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java
+++ b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java
@@ -28,6 +28,8 @@ import org.apache.druid.math.expr.InputBindings;
 import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor;
 import org.apache.druid.math.expr.vector.ExprVectorProcessor;
 import org.apache.druid.math.expr.vector.LongOutLongInFunctionVectorValueProcessor;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.joda.time.Period;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -36,8 +38,28 @@ import java.util.Objects;
 
 public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
 {
+  public static String forQueryGranularity(Period period)
+  {
+    return FN_NAME + "(" + ColumnHolder.TIME_COLUMN_NAME + ",'" + period + "')";
+  }
+
   private static final String FN_NAME = "timestamp_floor";
 
+  private static PeriodGranularity computeGranularity(
+      final Expr expr,
+      final List<Expr> args,
+      final Expr.ObjectBinding bindings
+  )
+  {
+    return ExprUtils.toPeriodGranularity(
+        expr,
+        args.get(1),
+        args.size() > 2 ? args.get(2) : null,
+        args.size() > 3 ? args.get(3) : null,
+        bindings
+    );
+  }
+
   @Override
   public String name()
   {
@@ -56,21 +78,6 @@ public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
     }
   }
 
-  private static PeriodGranularity computeGranularity(
-      final Expr expr,
-      final List<Expr> args,
-      final Expr.ObjectBinding bindings
-  )
-  {
-    return ExprUtils.toPeriodGranularity(
-        expr,
-        args.get(1),
-        args.size() > 2 ? args.get(2) : null,
-        args.size() > 3 ? args.get(3) : null,
-        bindings
-    );
-  }
-
   public static class TimestampFloorExpr extends ExprMacroTable.BaseScalarMacroFunctionExpr
   {
     private final PeriodGranularity granularity;
diff --git a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java
index 99ad72f0ee9..955553d2dd3 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java
@@ -25,6 +25,7 @@ import com.google.common.collect.RangeSet;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.Cacheable;
 import org.apache.druid.query.extraction.ExtractionFn;
+import org.apache.druid.segment.CursorBuildSpec;
 
 import javax.annotation.Nullable;
 import java.util.Set;
@@ -76,7 +77,8 @@ public interface DimFilter extends Cacheable
    *
    * The Filter returned by this method across multiple calls must be the same object: parts of the query stack
    * compare Filters, and returning the same object allows these checks to avoid deep comparisons.
-   * (see {@link org.apache.druid.segment.join.HashJoinSegmentStorageAdapter#makeCursors for an example}
+   * (see {@link org.apache.druid.segment.join.HashJoinSegmentStorageAdapter#makeCursorHolder(CursorBuildSpec)}
+   * for an example}
    *
    * @param mayIncludeUnknown whether the optimized filter may need to operate in "includeUnknown" mode.
    *                          See {@link NullHandling#useThreeValueLogic()}.
diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java
index 9fae7d51d20..f30681c8669 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java
@@ -106,7 +106,8 @@ public interface Filter
       matcherBundle = new FilterBundle.SimpleMatcherBundle(
           new FilterBundle.MatcherBundleInfo(this::toString, null, null),
           this::makeMatcher,
-          this::makeVectorMatcher
+          this::makeVectorMatcher,
+          this.canVectorizeMatcher(columnIndexSelector)
       );
     } else {
       matcherBundle = null;
diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterBundle.java b/processing/src/main/java/org/apache/druid/query/filter/FilterBundle.java
index 695eb26c00b..e105b0d6163 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/FilterBundle.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/FilterBundle.java
@@ -118,6 +118,11 @@ public class FilterBundle
     return matcherBundle != null;
   }
 
+  public boolean canVectorizeMatcher()
+  {
+    return matcherBundle == null || matcherBundle.canVectorize();
+  }
+
   public interface IndexBundle
   {
     IndexBundleInfo getIndexInfo();
@@ -142,6 +147,8 @@ public class FilterBundle
     ValueMatcher valueMatcher(ColumnSelectorFactory selectorFactory, Offset baseOffset, boolean descending);
 
     VectorValueMatcher vectorMatcher(VectorColumnSelectorFactory selectorFactory, ReadableVectorOffset baseOffset);
+
+    boolean canVectorize();
   }
 
   public static class SimpleIndexBundle implements IndexBundle
@@ -182,15 +189,19 @@ public class FilterBundle
     private final Function<ColumnSelectorFactory, ValueMatcher> matcherFn;
     private final Function<VectorColumnSelectorFactory, VectorValueMatcher> vectorMatcherFn;
 
+    private final boolean canVectorize;
+
     public SimpleMatcherBundle(
         MatcherBundleInfo matcherInfo,
         Function<ColumnSelectorFactory, ValueMatcher> matcherFn,
-        Function<VectorColumnSelectorFactory, VectorValueMatcher> vectorMatcherFn
+        Function<VectorColumnSelectorFactory, VectorValueMatcher> vectorMatcherFn,
+        boolean canVectorize
     )
     {
       this.matcherInfo = Preconditions.checkNotNull(matcherInfo);
       this.matcherFn = Preconditions.checkNotNull(matcherFn);
       this.vectorMatcherFn = Preconditions.checkNotNull(vectorMatcherFn);
+      this.canVectorize = canVectorize;
     }
 
     @Override
@@ -217,6 +228,12 @@ public class FilterBundle
     {
       return vectorMatcherFn.apply(selectorFactory);
     }
+
+    @Override
+    public boolean canVectorize()
+    {
+      return canVectorize;
+    }
   }
 
   public static class BundleInfo
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
index 994705f55e3..761aa43b1b3 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
@@ -74,7 +74,6 @@ import org.joda.time.Interval;
 import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
@@ -109,6 +108,8 @@ public class GroupByQuery extends BaseQuery<ResultRow>
   @Nullable
   private final DimFilter dimFilter;
   private final List<DimensionSpec> dimensions;
+  private final List<String> groupingColumns;
+
   private final List<AggregatorFactory> aggregatorSpecs;
   private final List<PostAggregator> postAggregatorSpecs;
   @Nullable
@@ -206,8 +207,10 @@ public class GroupByQuery extends BaseQuery<ResultRow>
     this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns);
     this.dimFilter = dimFilter;
     this.dimensions = dimensions == null ? ImmutableList.of() : dimensions;
+    this.groupingColumns = new ArrayList<>();
     for (DimensionSpec spec : this.dimensions) {
       Preconditions.checkArgument(spec != null, "dimensions has null DimensionSpec");
+      groupingColumns.add(spec.getDimension());
     }
 
     this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.of() : aggregatorSpecs;
@@ -795,12 +798,17 @@ public class GroupByQuery extends BaseQuery<ResultRow>
     return Queries.computeRequiredColumns(
         virtualColumns,
         dimFilter,
-        dimensions,
-        aggregatorSpecs,
-        Collections.emptyList()
+        groupingColumns,
+        aggregatorSpecs
     );
   }
 
+  @JsonIgnore
+  public List<String> getGroupingColumns()
+  {
+    return groupingColumns;
+  }
+
   @Override
   public GroupByQuery withOverriddenContext(Map<String, Object> contextOverride)
   {
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupingEngine.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupingEngine.java
index ab1ee1052b4..75d7ad50d8a 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/GroupingEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupingEngine.java
@@ -47,11 +47,13 @@ import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.LazySequence;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.query.DruidProcessingConfig;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryCapacityExceededException;
 import org.apache.druid.query.QueryContext;
 import org.apache.druid.query.QueryContexts;
+import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryProcessingPool;
 import org.apache.druid.query.QueryRunner;
@@ -63,7 +65,6 @@ import org.apache.druid.query.aggregation.PostAggregator;
 import org.apache.druid.query.context.ResponseContext;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.DimensionSpec;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.groupby.epinephelinae.BufferArrayGrouper;
 import org.apache.druid.query.groupby.epinephelinae.GroupByMergingQueryRunner;
 import org.apache.druid.query.groupby.epinephelinae.GroupByQueryEngine;
@@ -74,6 +75,9 @@ import org.apache.druid.query.groupby.orderby.DefaultLimitSpec;
 import org.apache.druid.query.groupby.orderby.LimitSpec;
 import org.apache.druid.query.groupby.orderby.NoopLimitSpec;
 import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
+import org.apache.druid.segment.ColumnInspector;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionHandlerUtils;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
@@ -486,6 +490,8 @@ public class GroupingEngine
 
     final ResourceHolder<ByteBuffer> bufferHolder = bufferPool.take();
 
+    Closer closer = Closer.create();
+    closer.register(bufferHolder);
     try {
       final String fudgeTimestampString = NullHandling.emptyToNullIfNeeded(
           query.context().getString(GroupingEngine.CTX_KEY_FUDGE_TIMESTAMP)
@@ -495,45 +501,44 @@ public class GroupingEngine
                                       ? null
                                       : DateTimes.utc(Long.parseLong(fudgeTimestampString));
 
-      final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter()));
-      final Interval interval = Iterables.getOnlyElement(query.getIntervals());
+      final CursorBuildSpec buildSpec = makeCursorBuildSpec(query, groupByQueryMetrics);
+      final CursorHolder cursorHolder = closer.register(storageAdapter.makeCursorHolder(buildSpec));
 
-      final boolean doVectorize = query.context().getVectorize().shouldVectorize(
-          VectorGroupByEngine.canVectorize(query, storageAdapter, filter)
-      );
+      final ColumnInspector inspector = query.getVirtualColumns().wrapInspector(storageAdapter);
 
+      // group by specific vectorization check
+      final boolean canVectorize = cursorHolder.canVectorize() &&
+                                   VectorGroupByEngine.canVectorizeDimensions(inspector, query.getDimensions());
+      final boolean shouldVectorize = query.context().getVectorize().shouldVectorize(canVectorize);
       final Sequence<ResultRow> result;
-
-      if (doVectorize) {
+      if (shouldVectorize) {
         result = VectorGroupByEngine.process(
             query,
             storageAdapter,
+            cursorHolder,
             bufferHolder.get(),
             fudgeTimestamp,
-            filter,
-            interval,
+            buildSpec.getInterval(),
             querySpecificConfig,
-            processingConfig,
-            groupByQueryMetrics
+            processingConfig
         );
       } else {
         result = GroupByQueryEngine.process(
             query,
             storageAdapter,
+            cursorHolder,
+            buildSpec,
             bufferHolder.get(),
             fudgeTimestamp,
             querySpecificConfig,
-            processingConfig,
-            filter,
-            interval,
-            groupByQueryMetrics
+            processingConfig
         );
       }
 
-      return result.withBaggage(bufferHolder);
+      return result.withBaggage(closer);
     }
     catch (Throwable e) {
-      bufferHolder.close();
+      CloseableUtils.closeAndWrapExceptions(closer);
       throw e;
     }
   }
@@ -840,6 +845,22 @@ public class GroupingEngine
     return aggsAndPostAggs;
   }
 
+  public static CursorBuildSpec makeCursorBuildSpec(GroupByQuery query, @Nullable QueryMetrics<?> queryMetrics)
+  {
+    return Granularities.decorateCursorBuildSpec(
+        query,
+        CursorBuildSpec.builder()
+                       .setInterval(query.getSingleInterval())
+                       .setFilter(Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter())))
+                       .setVirtualColumns(query.getVirtualColumns())
+                       .setGroupingColumns(query.getGroupingColumns())
+                       .setAggregators(query.getAggregatorSpecs())
+                       .setQueryContext(query.context())
+                       .setQueryMetrics(queryMetrics)
+                       .build()
+    );
+  }
+
 
   /**
    * Returns the cardinality of array needed to do array-based aggregation, or -1 if array-based aggregation
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByQueryEngine.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByQueryEngine.java
index 35f09c5446d..9f250772462 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByQueryEngine.java
@@ -26,15 +26,17 @@ import com.google.common.collect.ImmutableList;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.BaseSequence;
 import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.ColumnSelectorPlus;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.DruidProcessingConfig;
 import org.apache.druid.query.aggregation.AggregatorAdapters;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.dimension.DimensionSpec;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
 import org.apache.druid.query.groupby.GroupByQueryMetrics;
@@ -48,6 +50,8 @@ import org.apache.druid.query.ordering.StringComparator;
 import org.apache.druid.segment.ColumnInspector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionHandlerUtils;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.StorageAdapter;
@@ -56,7 +60,6 @@ import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.data.IndexedInts;
 import org.joda.time.DateTime;
-import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
 import java.io.Closeable;
@@ -94,89 +97,141 @@ public class GroupByQueryEngine
   public static Sequence<ResultRow> process(
       final GroupByQuery query,
       final StorageAdapter storageAdapter,
+      final CursorHolder cursorHolder,
+      final CursorBuildSpec buildSpec,
       final ByteBuffer processingBuffer,
       @Nullable final DateTime fudgeTimestamp,
       final GroupByQueryConfig querySpecificConfig,
-      final DruidProcessingConfig processingConfig,
-      @Nullable final Filter filter,
-      final Interval interval,
-      @Nullable final GroupByQueryMetrics groupByQueryMetrics
+      final DruidProcessingConfig processingConfig
   )
   {
-    final Sequence<Cursor> cursors = storageAdapter.makeCursors(
-        filter,
-        interval,
-        query.getVirtualColumns(),
+    final Cursor cursor = cursorHolder.asCursor();
+    if (cursor == null) {
+      return Sequences.empty();
+    }
+    final CursorGranularizer granularizer = CursorGranularizer.create(
+        storageAdapter,
+        cursor,
         query.getGranularity(),
-        false,
-        groupByQueryMetrics
+        buildSpec.getInterval(),
+        false
+    );
+    if (granularizer == null) {
+      return Sequences.empty();
+    }
+    final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+    final ColumnSelectorPlus<GroupByColumnSelectorStrategy>[] selectorPlus = DimensionHandlerUtils
+        .createColumnSelectorPluses(
+            STRATEGY_FACTORY,
+            query.getDimensions(),
+            columnSelectorFactory
+        );
+    GroupByColumnSelectorPlus[] dims = new GroupByColumnSelectorPlus[selectorPlus.length];
+    int curPos = 0;
+    for (int i = 0; i < dims.length; i++) {
+      dims[i] = new GroupByColumnSelectorPlus(
+          selectorPlus[i],
+          curPos,
+          query.getResultRowDimensionStart() + i
+      );
+      curPos += dims[i].getColumnSelectorStrategy().getGroupingKeySizeBytes();
+    }
+
+    final int cardinalityForArrayAggregation = GroupingEngine.getCardinalityForArrayAggregation(
+        querySpecificConfig,
+        query,
+        storageAdapter,
+        processingBuffer
     );
 
-    return cursors.flatMap(
-        cursor -> new BaseSequence<>(
-            new BaseSequence.IteratorMaker<ResultRow, GroupByEngineIterator<?>>()
-            {
-              @Override
-              public GroupByEngineIterator<?> make()
-              {
-                final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
-                final ColumnSelectorPlus<GroupByColumnSelectorStrategy>[] selectorPlus = DimensionHandlerUtils
-                    .createColumnSelectorPluses(
-                        STRATEGY_FACTORY,
-                        query.getDimensions(),
-                        columnSelectorFactory
-                    );
-                GroupByColumnSelectorPlus[] dims = new GroupByColumnSelectorPlus[selectorPlus.length];
-                int curPos = 0;
-                for (int i = 0; i < dims.length; i++) {
-                  dims[i] = new GroupByColumnSelectorPlus(
-                      selectorPlus[i],
-                      curPos,
-                      query.getResultRowDimensionStart() + i
-                  );
-                  curPos += dims[i].getColumnSelectorStrategy().getGroupingKeySizeBytes();
-                }
+    if (query.getGranularity().equals(Granularities.ALL)) {
+      return makeEngineIteratorSequence(
+          query,
+          processingBuffer,
+          fudgeTimestamp,
+          querySpecificConfig,
+          processingConfig,
+          cardinalityForArrayAggregation,
+          cursor,
+          granularizer,
+          dims,
+          columnSelectorFactory
+      );
+    }
 
-                final int cardinalityForArrayAggregation = GroupingEngine.getCardinalityForArrayAggregation(
-                    querySpecificConfig,
-                    query,
-                    storageAdapter,
-                    processingBuffer
-                );
+    return Sequences.simple(granularizer.getBucketIterable())
+                    .flatMap(bucketInterval -> {
+                      if (!granularizer.advanceToBucket(bucketInterval)) {
+                        return Sequences.empty();
+                      }
+                      return makeEngineIteratorSequence(
+                          query,
+                          processingBuffer,
+                          fudgeTimestamp,
+                          querySpecificConfig,
+                          processingConfig,
+                          cardinalityForArrayAggregation,
+                          cursor,
+                          granularizer,
+                          dims,
+                          columnSelectorFactory
+                      );
+                    });
+  }
 
-                if (cardinalityForArrayAggregation >= 0) {
-                  return new ArrayAggregateIterator(
-                      query,
-                      querySpecificConfig,
-                      processingConfig,
-                      cursor,
-                      processingBuffer,
-                      fudgeTimestamp,
-                      dims,
-                      hasNoImplicitUnnestDimensions(columnSelectorFactory, query.getDimensions()),
-                      cardinalityForArrayAggregation
-                  );
-                } else {
-                  return new HashAggregateIterator(
-                      query,
-                      querySpecificConfig,
-                      processingConfig,
-                      cursor,
-                      processingBuffer,
-                      fudgeTimestamp,
-                      dims,
-                      hasNoImplicitUnnestDimensions(columnSelectorFactory, query.getDimensions())
-                  );
-                }
-              }
-
-              @Override
-              public void cleanup(GroupByEngineIterator<?> iterFromMake)
-              {
-                iterFromMake.close();
-              }
+  private static Sequence<ResultRow> makeEngineIteratorSequence(
+      GroupByQuery query,
+      ByteBuffer processingBuffer,
+      @Nullable DateTime fudgeTimestamp,
+      GroupByQueryConfig querySpecificConfig,
+      DruidProcessingConfig processingConfig,
+      int cardinalityForArrayAggregation,
+      Cursor cursor,
+      CursorGranularizer granularizer,
+      GroupByColumnSelectorPlus[] dims,
+      ColumnSelectorFactory columnSelectorFactory
+  )
+  {
+    return new BaseSequence<>(
+        new BaseSequence.IteratorMaker<ResultRow, GroupByEngineIterator<?>>()
+        {
+          @Override
+          public GroupByEngineIterator<?> make()
+          {
+            if (cardinalityForArrayAggregation >= 0) {
+              return new ArrayAggregateIterator(
+                  query,
+                  querySpecificConfig,
+                  processingConfig,
+                  cursor,
+                  granularizer,
+                  processingBuffer,
+                  fudgeTimestamp,
+                  dims,
+                  hasNoImplicitUnnestDimensions(columnSelectorFactory, query.getDimensions()),
+                  cardinalityForArrayAggregation
+              );
+            } else {
+              return new HashAggregateIterator(
+                  query,
+                  querySpecificConfig,
+                  processingConfig,
+                  cursor,
+                  granularizer,
+                  processingBuffer,
+                  fudgeTimestamp,
+                  dims,
+                  hasNoImplicitUnnestDimensions(columnSelectorFactory, query.getDimensions())
+              );
             }
-        )
+          }
+
+          @Override
+          public void cleanup(GroupByEngineIterator<?> iterFromMake)
+          {
+            iterFromMake.close();
+          }
+        }
     );
   }
 
@@ -237,6 +292,7 @@ public class GroupByQueryEngine
     protected final GroupByQuery query;
     protected final GroupByQueryConfig querySpecificConfig;
     protected final Cursor cursor;
+    protected final CursorGranularizer granularizer;
     protected final ByteBuffer buffer;
     protected final Grouper.KeySerde<ByteBuffer> keySerde;
     protected final GroupByColumnSelectorPlus[] dims;
@@ -253,6 +309,7 @@ public class GroupByQueryEngine
         final GroupByQueryConfig querySpecificConfig,
         final DruidProcessingConfig processingConfig,
         final Cursor cursor,
+        final CursorGranularizer granularizer,
         final ByteBuffer buffer,
         @Nullable final DateTime fudgeTimestamp,
         final GroupByColumnSelectorPlus[] dims,
@@ -263,12 +320,13 @@ public class GroupByQueryEngine
       this.querySpecificConfig = querySpecificConfig;
       this.maxSelectorFootprint = querySpecificConfig.getActualMaxSelectorDictionarySize(processingConfig);
       this.cursor = cursor;
+      this.granularizer = granularizer;
       this.buffer = buffer;
       this.keySerde = new GroupByEngineKeySerde(dims, query);
       this.dims = dims;
 
       // Time is the same for every row in the cursor
-      this.timestamp = fudgeTimestamp != null ? fudgeTimestamp : cursor.getTime();
+      this.timestamp = fudgeTimestamp != null ? fudgeTimestamp : granularizer.getBucketStart();
       this.allSingleValueDims = allSingleValueDims;
       this.allowMultiValueGrouping = query.context().getBoolean(
           GroupByQueryConfig.CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING,
@@ -332,7 +390,7 @@ public class GroupByQueryEngine
       if (delegate != null && delegate.hasNext()) {
         return true;
       } else {
-        if (!cursor.isDone()) {
+        if (!cursor.isDone() && granularizer.currentOffsetWithinBucket()) {
           if (delegate != null) {
             delegate.close();
           }
@@ -417,13 +475,14 @@ public class GroupByQueryEngine
         GroupByQueryConfig querySpecificConfig,
         DruidProcessingConfig processingConfig,
         Cursor cursor,
+        CursorGranularizer granularizer,
         ByteBuffer buffer,
         @Nullable DateTime fudgeTimestamp,
         GroupByColumnSelectorPlus[] dims,
         boolean allSingleValueDims
     )
     {
-      super(query, querySpecificConfig, processingConfig, cursor, buffer, fudgeTimestamp, dims, allSingleValueDims);
+      super(query, querySpecificConfig, processingConfig, cursor, granularizer, buffer, fudgeTimestamp, dims, allSingleValueDims);
 
       final int dimCount = query.getDimensions().size();
       stack = new int[dimCount];
@@ -527,7 +586,9 @@ public class GroupByQueryEngine
           return;
         }
 
-        cursor.advance();
+        if (!granularizer.advanceCursorWithinBucket()) {
+          break;
+        }
 
         // Check selectorInternalFootprint after advancing the cursor. (We reset after the first row that causes
         // us to go past the limit.)
@@ -615,7 +676,9 @@ public class GroupByQueryEngine
         }
 
         // Advance to next row
-        cursor.advance();
+        if (!granularizer.advanceCursorWithinBucket()) {
+          break;
+        }
         currentRowWasPartiallyAggregated = false;
 
         // Check selectorInternalFootprint after advancing the cursor. (We reset after the first row that causes
@@ -656,6 +719,7 @@ public class GroupByQueryEngine
         GroupByQueryConfig querySpecificConfig,
         DruidProcessingConfig processingConfig,
         Cursor cursor,
+        CursorGranularizer granularizer,
         ByteBuffer buffer,
         @Nullable DateTime fudgeTimestamp,
         GroupByColumnSelectorPlus[] dims,
@@ -663,7 +727,7 @@ public class GroupByQueryEngine
         int cardinality
     )
     {
-      super(query, querySpecificConfig, processingConfig, cursor, buffer, fudgeTimestamp, dims, allSingleValueDims);
+      super(query, querySpecificConfig, processingConfig, cursor, granularizer, buffer, fudgeTimestamp, dims, allSingleValueDims);
       this.cardinality = cardinality;
       if (dims.length == 1) {
         this.dim = dims[0];
@@ -713,7 +777,9 @@ public class GroupByQueryEngine
         if (!grouper.aggregate(key).isOk()) {
           return;
         }
-        cursor.advance();
+        if (!granularizer.advanceCursorWithinBucket()) {
+          break;
+        }
       }
     }
 
@@ -751,7 +817,9 @@ public class GroupByQueryEngine
           }
         }
 
-        cursor.advance();
+        if (!granularizer.advanceCursorWithinBucket()) {
+          break;
+        }
         if (!cursor.isDone()) {
           // dim is always an indexed string dimension
           multiValues = ((DimensionSelector) dim.getSelector()).getRow();
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
index 75f4539e8c3..4d0a30547c2 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
@@ -22,17 +22,14 @@ package org.apache.druid.query.groupby.epinephelinae.vector;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Suppliers;
 import org.apache.datasketches.memory.WritableMemory;
-import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.guava.BaseSequence;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.query.DruidProcessingConfig;
 import org.apache.druid.query.aggregation.AggregatorAdapters;
-import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.DimensionSpec;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
 import org.apache.druid.query.groupby.GroupByQueryMetrics;
@@ -47,10 +44,9 @@ import org.apache.druid.query.groupby.epinephelinae.collection.MemoryPointer;
 import org.apache.druid.query.vector.VectorCursorGranularizer;
 import org.apache.druid.segment.ColumnInspector;
 import org.apache.druid.segment.ColumnProcessors;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
-import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
 import org.apache.druid.segment.vector.VectorCursor;
 import org.joda.time.DateTime;
@@ -84,33 +80,21 @@ public class VectorGroupByEngine
   public static Sequence<ResultRow> process(
       final GroupByQuery query,
       final StorageAdapter storageAdapter,
+      final CursorHolder cursorHolder,
       final ByteBuffer processingBuffer,
       @Nullable final DateTime fudgeTimestamp,
-      @Nullable final Filter filter,
       final Interval interval,
       final GroupByQueryConfig config,
-      final DruidProcessingConfig processingConfig,
-      @Nullable final GroupByQueryMetrics groupByQueryMetrics
+      final DruidProcessingConfig processingConfig
   )
   {
-    if (!canVectorize(query, storageAdapter, filter)) {
-      throw new ISE("Cannot vectorize");
-    }
-
     return new BaseSequence<>(
         new BaseSequence.IteratorMaker<ResultRow, CloseableIterator<ResultRow>>()
         {
           @Override
           public CloseableIterator<ResultRow> make()
           {
-            final VectorCursor cursor = storageAdapter.makeVectorCursor(
-                Filters.toFilter(query.getDimFilter()),
-                interval,
-                query.getVirtualColumns(),
-                false,
-                query.context().getVectorSize(),
-                groupByQueryMetrics
-            );
+            final VectorCursor cursor = cursorHolder.asVectorCursor();
 
             if (cursor == null) {
               // Return empty iterator.
@@ -136,48 +120,37 @@ public class VectorGroupByEngine
               };
             }
 
-            try {
-              final VectorColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
-              final List<GroupByVectorColumnSelector> dimensions = query.getDimensions().stream().map(
-                  dimensionSpec -> {
-                    if (dimensionSpec instanceof DefaultDimensionSpec) {
-                      // Delegate creation of GroupByVectorColumnSelector to the column selector factory, so that
-                      // virtual columns (like ExpressionVirtualColumn) can control their own grouping behavior.
-                      return columnSelectorFactory.makeGroupByVectorColumnSelector(
-                          dimensionSpec.getDimension(),
-                          config.getDeferExpressionDimensions()
-                      );
-                    } else {
-                      return ColumnProcessors.makeVectorProcessor(
-                          dimensionSpec,
-                          GroupByVectorColumnProcessorFactory.instance(),
-                          columnSelectorFactory
-                      );
-                    }
+            final VectorColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+            final List<GroupByVectorColumnSelector> dimensions = query.getDimensions().stream().map(
+                dimensionSpec -> {
+                  if (dimensionSpec instanceof DefaultDimensionSpec) {
+                    // Delegate creation of GroupByVectorColumnSelector to the column selector factory, so that
+                    // virtual columns (like ExpressionVirtualColumn) can control their own grouping behavior.
+                    return columnSelectorFactory.makeGroupByVectorColumnSelector(
+                        dimensionSpec.getDimension(),
+                        config.getDeferExpressionDimensions()
+                    );
+                  } else {
+                    return ColumnProcessors.makeVectorProcessor(
+                        dimensionSpec,
+                        GroupByVectorColumnProcessorFactory.instance(),
+                        columnSelectorFactory
+                    );
                   }
-              ).collect(Collectors.toList());
+                }
+            ).collect(Collectors.toList());
 
-              return new VectorGroupByEngineIterator(
-                  query,
-                  config,
-                  processingConfig,
-                  storageAdapter,
-                  cursor,
-                  interval,
-                  dimensions,
-                  processingBuffer,
-                  fudgeTimestamp
-              );
-            }
-            catch (Throwable e) {
-              try {
-                cursor.close();
-              }
-              catch (Throwable e2) {
-                e.addSuppressed(e2);
-              }
-              throw e;
-            }
+            return new VectorGroupByEngineIterator(
+                query,
+                config,
+                processingConfig,
+                storageAdapter,
+                cursor,
+                interval,
+                dimensions,
+                processingBuffer,
+                fudgeTimestamp
+            );
           }
 
           @Override
@@ -194,21 +167,7 @@ public class VectorGroupByEngine
     );
   }
 
-  public static boolean canVectorize(
-      final GroupByQuery query,
-      final StorageAdapter adapter,
-      @Nullable final Filter filter
-  )
-  {
-    final ColumnInspector inspector = query.getVirtualColumns().wrapInspector(adapter);
-
-    return adapter.canVectorize(filter, query.getVirtualColumns(), false)
-           && canVectorizeDimensions(inspector, query.getDimensions())
-           && VirtualColumns.shouldVectorize(query, query.getVirtualColumns(), adapter)
-           && canVectorizeAggregators(inspector, query.getAggregatorSpecs());
-  }
-
-  private static boolean canVectorizeDimensions(
+  public static boolean canVectorizeDimensions(
       final ColumnInspector inspector,
       final List<DimensionSpec> dimensions
   )
@@ -241,19 +200,6 @@ public class VectorGroupByEngine
     return true;
   }
 
-  public static boolean canVectorizeAggregators(
-      final ColumnInspector inspector,
-      final List<AggregatorFactory> aggregatorFactories
-  )
-  {
-    for (AggregatorFactory aggregatorFactory : aggregatorFactories) {
-      if (!aggregatorFactory.canVectorize(inspector)) {
-        return false;
-      }
-    }
-    return true;
-  }
-
   @VisibleForTesting
   static class VectorGroupByEngineIterator implements CloseableIterator<ResultRow>
   {
@@ -366,7 +312,6 @@ public class VectorGroupByEngine
       if (delegate != null) {
         closer.register(delegate);
       }
-      closer.register(cursor);
       closer.close();
     }
 
diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
index 81fcf34ed01..6d0629c4dce 100644
--- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
+++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
@@ -23,19 +23,17 @@ import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Accumulator;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.metadata.metadata.ColumnAnalysis;
 import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.BaseColumn;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
@@ -52,8 +50,6 @@ import org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter;
 import org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
 import org.apache.druid.segment.serde.ComplexMetrics;
-import org.joda.time.DateTime;
-import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
 import java.io.IOException;
@@ -274,48 +270,25 @@ public class SegmentAnalyzer
     }
 
     if (analyzingSize()) {
-      final DateTime start = storageAdapter.getMinTime();
-      final DateTime end = storageAdapter.getMaxTime();
+      try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        final Cursor cursor = cursorHolder.asCursor();
 
-      final Sequence<Cursor> cursors =
-          storageAdapter.makeCursors(
-              null,
-              new Interval(start, end),
-              VirtualColumns.EMPTY,
-              Granularities.ALL,
-              false,
-              null
-          );
-
-      size = cursors.accumulate(
-          0L,
-          new Accumulator<Long, Cursor>()
-          {
-            @Override
-            public Long accumulate(Long accumulated, Cursor cursor)
-            {
-              DimensionSelector selector = cursor
-                  .getColumnSelectorFactory()
-                  .makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName));
-              if (selector == null) {
-                return accumulated;
+        if (cursor != null) {
+          final DimensionSelector selector =
+              cursor.getColumnSelectorFactory()
+                    .makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName));
+          while (!cursor.isDone()) {
+            final IndexedInts row = selector.getRow();
+            for (int i = 0, rowSize = row.size(); i < rowSize; ++i) {
+              final String dimVal = selector.lookupName(row.get(i));
+              if (dimVal != null && !dimVal.isEmpty()) {
+                size += StringUtils.estimatedBinaryLengthAsUTF8(dimVal);
               }
-              long current = accumulated;
-              while (!cursor.isDone()) {
-                final IndexedInts row = selector.getRow();
-                for (int i = 0, rowSize = row.size(); i < rowSize; ++i) {
-                  final String dimVal = selector.lookupName(row.get(i));
-                  if (dimVal != null && !dimVal.isEmpty()) {
-                    current += StringUtils.estimatedBinaryLengthAsUTF8(dimVal);
-                  }
-                }
-                cursor.advance();
-              }
-
-              return current;
             }
+            cursor.advance();
           }
-      );
+        }
+      }
     }
 
     if (analyzingMinMax()) {
diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java
index 85c84fe3858..f2d434bab8a 100644
--- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java
@@ -97,7 +97,7 @@ public class SegmentMetadataQuery extends BaseQuery<SegmentAnalysis>
       @JsonProperty("aggregatorMergeStrategy") AggregatorMergeStrategy aggregatorMergeStrategy
   )
   {
-    super(dataSource, querySegmentSpec == null ? DEFAULT_SEGMENT_SPEC : querySegmentSpec, false, context);
+    super(dataSource, querySegmentSpec == null ? DEFAULT_SEGMENT_SPEC : querySegmentSpec, context);
 
     if (querySegmentSpec == null) {
       this.usingDefaultInterval = true;
diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java
index 50289b9851b..aed132b745e 100644
--- a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java
@@ -25,6 +25,8 @@ import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.DataSource;
 import org.apache.druid.query.InlineDataSource;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.filter.DimFilter;
@@ -34,7 +36,6 @@ import org.apache.druid.query.spec.QuerySegmentSpec;
 import org.apache.druid.segment.column.RowSignature;
 
 import javax.annotation.Nullable;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -97,7 +98,6 @@ public class WindowOperatorQuery extends BaseQuery<RowsAndColumns>
     super(
         validateMaybeRewriteDataSource(dataSource, leafOperators != null),
         intervals,
-        false,
         context
     );
     this.rowSignature = rowSignature;
@@ -114,11 +114,11 @@ public class WindowOperatorQuery extends BaseQuery<RowsAndColumns>
           ScanQuery scan = (ScanQuery) subQuery;
 
           ArrayList<ColumnWithDirection> ordering = new ArrayList<>();
-          for (ScanQuery.OrderBy orderBy : scan.getOrderBys()) {
+          for (OrderBy orderBy : scan.getOrderBys()) {
             ordering.add(
                 new ColumnWithDirection(
                     orderBy.getColumnName(),
-                    ScanQuery.Order.DESCENDING == orderBy.getOrder()
+                    Order.DESCENDING == orderBy.getOrder()
                     ? ColumnWithDirection.Direction.DESC
                     : ColumnWithDirection.Direction.ASC
                 )
diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/LazilyDecoratedRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/LazilyDecoratedRowsAndColumns.java
index ce199a7803c..6be9500b9fe 100644
--- a/processing/src/main/java/org/apache/druid/query/rowsandcols/LazilyDecoratedRowsAndColumns.java
+++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/LazilyDecoratedRowsAndColumns.java
@@ -30,11 +30,8 @@ import org.apache.druid.frame.write.FrameWriter;
 import org.apache.druid.frame.write.FrameWriterFactory;
 import org.apache.druid.frame.write.FrameWriters;
 import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.UOE;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.query.operator.ColumnWithDirection;
@@ -48,6 +45,8 @@ import org.apache.druid.query.rowsandcols.semantic.RowsAndColumnsDecorator;
 import org.apache.druid.query.rowsandcols.semantic.WireTransferable;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
@@ -208,16 +207,6 @@ public class LazilyDecoratedRowsAndColumns implements RowsAndColumns
   @Nullable
   private Pair<byte[], RowSignature> materializeStorageAdapter(StorageAdapter as)
   {
-    final Sequence<Cursor> cursors = as.makeCursors(
-        filter,
-        interval == null ? Intervals.ETERNITY : interval,
-        virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-
     final Collection<String> cols;
     if (viewableColumns != null) {
       cols = viewableColumns;
@@ -231,19 +220,27 @@ public class LazilyDecoratedRowsAndColumns implements RowsAndColumns
                             .build();
       }
     }
-    AtomicReference<RowSignature> siggy = new AtomicReference<>(null);
+    final CursorBuildSpec.CursorBuildSpecBuilder builder = CursorBuildSpec.builder()
+                                                                          .setFilter(filter);
+    if (interval != null) {
+      builder.setInterval(interval);
+    }
+    if (virtualColumns != null) {
+      builder.setVirtualColumns(virtualColumns);
+    }
+    try (final CursorHolder cursorHolder = as.makeCursorHolder(builder.build())) {
+      final Cursor cursor = cursorHolder.asCursor();
 
-    FrameWriter writer = cursors.accumulate(null, (accumulated, in) -> {
-      if (accumulated != null) {
-        // We should not get multiple cursors because we set the granularity to ALL.  So, this should never
-        // actually happen, but it doesn't hurt us to defensive here, so we test against it.
-        throw new ISE("accumulated[%s] non-null, why did we get multiple cursors?", accumulated);
+      if (cursor == null) {
+        return null;
       }
 
+      final AtomicReference<RowSignature> siggy = new AtomicReference<>(null);
+
       long remainingRowsToSkip = limit.getOffset();
       long remainingRowsToFetch = limit.getLimitOrMax();
 
-      final ColumnSelectorFactory columnSelectorFactory = in.getColumnSelectorFactory();
+      final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
       final RowSignature.Builder sigBob = RowSignature.builder();
 
       for (String col : cols) {
@@ -282,26 +279,24 @@ public class LazilyDecoratedRowsAndColumns implements RowsAndColumns
           sortColumns
       );
 
-      final FrameWriter frameWriter = frameWriterFactory.newFrameWriter(columnSelectorFactory);
-      for (; !in.isDoneOrInterrupted() && remainingRowsToSkip > 0; remainingRowsToSkip--) {
-        in.advance();
+      final FrameWriter writer = frameWriterFactory.newFrameWriter(columnSelectorFactory);
+      for (; !cursor.isDoneOrInterrupted() && remainingRowsToSkip > 0; remainingRowsToSkip--) {
+        cursor.advance();
       }
-      for (; !in.isDoneOrInterrupted() && remainingRowsToFetch > 0; remainingRowsToFetch--) {
-        frameWriter.addSelection();
-        in.advance();
+      for (; !cursor.isDoneOrInterrupted() && remainingRowsToFetch > 0; remainingRowsToFetch--) {
+        writer.addSelection();
+        cursor.advance();
       }
 
-      return frameWriter;
-    });
-
-    if (writer == null) {
-      // This means that the accumulate was never called, which can only happen if we didn't have any cursors.
-      // We would only have zero cursors if we essentially didn't match anything, meaning that our RowsAndColumns
-      // should be completely empty.
-      return null;
-    } else {
-      final byte[] bytes = writer.toByteArray();
-      return Pair.of(bytes, siggy.get());
+      if (writer == null) {
+        // This means that the accumulate was never called, which can only happen if we didn't have any cursors.
+        // We would only have zero cursors if we essentially didn't match anything, meaning that our RowsAndColumns
+        // should be completely empty.
+        return null;
+      } else {
+        final byte[] bytes = writer.toByteArray();
+        return Pair.of(bytes, siggy.get());
+      }
     }
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/StorageAdapterRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/StorageAdapterRowsAndColumns.java
index edc6518ca04..2d49cc32440 100644
--- a/processing/src/main/java/org/apache/druid/query/rowsandcols/StorageAdapterRowsAndColumns.java
+++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/StorageAdapterRowsAndColumns.java
@@ -24,17 +24,14 @@ import org.apache.druid.frame.allocation.ArenaMemoryAllocatorFactory;
 import org.apache.druid.frame.write.FrameWriter;
 import org.apache.druid.frame.write.FrameWriterFactory;
 import org.apache.druid.frame.write.FrameWriters;
-import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.rowsandcols.column.Column;
 import org.apache.druid.query.rowsandcols.concrete.ColumnBasedFrameRowsAndColumns;
 import org.apache.druid.segment.CloseableShapeshifter;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.RowSignature;
 
 import javax.annotation.Nonnull;
@@ -98,25 +95,16 @@ public class StorageAdapterRowsAndColumns implements CloseableShapeshifter, Rows
   @Nonnull
   private static RowsAndColumns materialize(StorageAdapter as)
   {
-    final Sequence<Cursor> cursors = as.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    try (final CursorHolder cursorHolder = as.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
 
-    RowSignature rowSignature = as.getRowSignature();
-
-    FrameWriter writer = cursors.accumulate(null, (accumulated, in) -> {
-      if (accumulated != null) {
-        // We should not get multiple cursors because we set the granularity to ALL.  So, this should never
-        // actually happen, but it doesn't hurt us to defensive here, so we test against it.
-        throw new ISE("accumulated[%s] non-null, why did we get multiple cursors?", accumulated);
+      if (cursor == null) {
+        return new EmptyRowsAndColumns();
       }
+      
+      final RowSignature rowSignature = as.getRowSignature();
 
-      final ColumnSelectorFactory columnSelectorFactory = in.getColumnSelectorFactory();
+      final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
       final FrameWriterFactory frameWriterFactory = FrameWriters.makeColumnBasedFrameWriterFactory(
           new ArenaMemoryAllocatorFactory(200 << 20), // 200 MB, because, why not?
@@ -124,19 +112,19 @@ public class StorageAdapterRowsAndColumns implements CloseableShapeshifter, Rows
           Collections.emptyList()
       );
 
-      final FrameWriter frameWriter = frameWriterFactory.newFrameWriter(columnSelectorFactory);
-      while (!in.isDoneOrInterrupted()) {
-        frameWriter.addSelection();
-        in.advance();
-      }
-      return frameWriter;
-    });
+      try (final FrameWriter writer = frameWriterFactory.newFrameWriter(columnSelectorFactory)) {
+        while (!cursor.isDoneOrInterrupted()) {
+          writer.addSelection();
+          cursor.advance();
+        }
 
-    if (writer == null) {
-      return new EmptyRowsAndColumns();
-    } else {
-      final byte[] bytes = writer.toByteArray();
-      return new ColumnBasedFrameRowsAndColumns(Frame.wrap(bytes), rowSignature);
+        if (writer == null) {
+          return new EmptyRowsAndColumns();
+        } else {
+          final byte[] bytes = writer.toByteArray();
+          return new ColumnBasedFrameRowsAndColumns(Frame.wrap(bytes), rowSignature);
+        }
+      }
     }
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java
index 96693a7b389..037148f50c3 100644
--- a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java
@@ -30,12 +30,13 @@ import com.google.common.collect.Ordering;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.UOE;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.DataSource;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.InlineDataSource;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Queries;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.operator.OffsetLimit;
@@ -96,83 +97,6 @@ public class ScanQuery extends BaseQuery<ScanResultValue>
     }
   }
 
-  public static class OrderBy
-  {
-    private final String columnName;
-    private final Order order;
-
-    @JsonCreator
-    public OrderBy(
-        @JsonProperty("columnName") final String columnName,
-        @JsonProperty("order") final Order order
-    )
-    {
-      this.columnName = Preconditions.checkNotNull(columnName, "columnName");
-      this.order = Preconditions.checkNotNull(order, "order");
-
-      if (order == Order.NONE) {
-        throw new IAE("Order required for column [%s]", columnName);
-      }
-    }
-
-    @JsonProperty
-    public String getColumnName()
-    {
-      return columnName;
-    }
-
-    @JsonProperty
-    public Order getOrder()
-    {
-      return order;
-    }
-
-    @Override
-    public boolean equals(Object o)
-    {
-      if (this == o) {
-        return true;
-      }
-      if (o == null || getClass() != o.getClass()) {
-        return false;
-      }
-      OrderBy that = (OrderBy) o;
-      return Objects.equals(columnName, that.columnName) && order == that.order;
-    }
-
-    @Override
-    public int hashCode()
-    {
-      return Objects.hash(columnName, order);
-    }
-
-    @Override
-    public String toString()
-    {
-      return StringUtils.format("%s %s", columnName, order == Order.ASCENDING ? "ASC" : "DESC");
-    }
-  }
-
-  public enum Order
-  {
-    ASCENDING,
-    DESCENDING,
-    NONE;
-
-    @JsonValue
-    @Override
-    public String toString()
-    {
-      return StringUtils.toLowerCase(this.name());
-    }
-
-    @JsonCreator
-    public static Order fromString(String name)
-    {
-      return valueOf(StringUtils.toUpperCase(name));
-    }
-  }
-
   /**
    * This context flag corresponds to whether the query is running on the "outermost" process (i.e. the process
    * the query is sent to).
@@ -210,7 +134,7 @@ public class ScanQuery extends BaseQuery<ScanResultValue>
       @JsonProperty("columnTypes") List<ColumnType> columnTypes
   )
   {
-    super(dataSource, querySegmentSpec, false, context);
+    super(dataSource, querySegmentSpec, context);
     this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns);
     this.resultFormat = (resultFormat == null) ? ResultFormat.RESULT_FORMAT_LIST : resultFormat;
     this.batchSize = (batchSize == 0) ? DEFAULT_BATCH_SIZE : batchSize;
@@ -484,9 +408,8 @@ public class ScanQuery extends BaseQuery<ScanResultValue>
       return Queries.computeRequiredColumns(
           virtualColumns,
           dimFilter,
-          Collections.emptyList(),
-          Collections.emptyList(),
-          columns
+          columns,
+          Collections.emptyList()
       );
     }
   }
diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
index 567e07fbdc7..893480f5453 100644
--- a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
@@ -26,16 +26,19 @@ import com.google.common.collect.Sets;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.UOE;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.BaseSequence;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.QueryTimeoutException;
 import org.apache.druid.query.context.ResponseContext;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumn;
@@ -67,7 +70,7 @@ public class ScanQueryEngine
   )
   {
     final Long numScannedRows = responseContext.getRowScanCount();
-    if (numScannedRows != null && numScannedRows >= query.getScanRowsLimit() && query.getTimeOrder().equals(ScanQuery.Order.NONE)) {
+    if (numScannedRows != null && numScannedRows >= query.getScanRowsLimit() && query.getTimeOrder().equals(Order.NONE)) {
       return Sequences.empty();
     }
     final boolean hasTimeout = query.context().hasTimeout();
@@ -112,127 +115,121 @@ public class ScanQueryEngine
 
     final SegmentId segmentId = segment.getId();
 
-    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter()));
-
     // If the row count is not set, set it to 0, else do nothing.
     responseContext.addRowScanCount(0);
     final long limit = calculateRemainingScanRowsLimit(query, responseContext);
-    return Sequences.concat(
-            adapter
-                .makeCursors(
-                    filter,
-                    intervals.get(0),
-                    query.getVirtualColumns(),
-                    Granularities.ALL,
-                    query.getTimeOrder().equals(ScanQuery.Order.DESCENDING) ||
-                    (query.getTimeOrder().equals(ScanQuery.Order.NONE) && query.isDescending()),
-                    queryMetrics
-                )
-                .map(cursor -> new BaseSequence<>(
-                    new BaseSequence.IteratorMaker<ScanResultValue, Iterator<ScanResultValue>>()
-                    {
-                      @Override
-                      public Iterator<ScanResultValue> make()
-                      {
-                        final List<BaseObjectColumnValueSelector> columnSelectors = new ArrayList<>(allColumns.size());
-                        final RowSignature.Builder rowSignatureBuilder = RowSignature.builder();
-                        final ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
+    final CursorHolder cursorHolder = adapter.makeCursorHolder(makeCursorBuildSpec(query, queryMetrics));
+    if (Order.NONE != query.getTimeOrder()) {
+      Cursors.requireTimeOrdering(cursorHolder, query.getTimeOrder());
+    }
+    return new BaseSequence<>(
+        new BaseSequence.IteratorMaker<ScanResultValue, Iterator<ScanResultValue>>()
+        {
+          @Override
+          public Iterator<ScanResultValue> make()
+          {
+            final Cursor cursor = cursorHolder.asCursor();
+            if (cursor == null) {
+              return Collections.emptyIterator();
+            }
+            final List<BaseObjectColumnValueSelector> columnSelectors = new ArrayList<>(allColumns.size());
+            final RowSignature.Builder rowSignatureBuilder = RowSignature.builder();
+            final ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
 
-                        for (String column : allColumns) {
-                          final BaseObjectColumnValueSelector selector = factory.makeColumnValueSelector(column);
-                          ColumnCapabilities columnCapabilities = factory.getColumnCapabilities(column);
-                          rowSignatureBuilder.add(
-                              column,
-                              columnCapabilities == null ? null : columnCapabilities.toColumnType()
-                          );
+            for (String column : allColumns) {
+              final BaseObjectColumnValueSelector selector = factory.makeColumnValueSelector(column);
+              ColumnCapabilities columnCapabilities = factory.getColumnCapabilities(column);
+              rowSignatureBuilder.add(
+                  column,
+                  columnCapabilities == null ? null : columnCapabilities.toColumnType()
+              );
 
-                          columnSelectors.add(selector);
-                        }
+              columnSelectors.add(selector);
+            }
 
-                        final int batchSize = query.getBatchSize();
-                        return new Iterator<ScanResultValue>()
-                        {
-                          private long offset = 0;
+            final int batchSize = query.getBatchSize();
+            return new Iterator<ScanResultValue>()
+            {
+              private long offset = 0;
 
-                          @Override
-                          public boolean hasNext()
-                          {
-                            return !cursor.isDone() && offset < limit;
-                          }
+              @Override
+              public boolean hasNext()
+              {
+                return !cursor.isDone() && offset < limit;
+              }
 
-                          @Override
-                          public ScanResultValue next()
-                          {
-                            if (!hasNext()) {
-                              throw new NoSuchElementException();
-                            }
-                            if (hasTimeout && System.currentTimeMillis() >= timeoutAt) {
-                              throw new QueryTimeoutException(StringUtils.nonStrictFormat("Query [%s] timed out", query.getId()));
-                            }
-                            final long lastOffset = offset;
-                            final Object events;
-                            final ScanQuery.ResultFormat resultFormat = query.getResultFormat();
-                            if (ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST.equals(resultFormat)) {
-                              events = rowsToCompactedList();
-                            } else if (ScanQuery.ResultFormat.RESULT_FORMAT_LIST.equals(resultFormat)) {
-                              events = rowsToList();
-                            } else {
-                              throw new UOE("resultFormat[%s] is not supported", resultFormat.toString());
-                            }
-                            responseContext.addRowScanCount(offset - lastOffset);
-                            return new ScanResultValue(segmentId.toString(), allColumns, events, rowSignatureBuilder.build());
-                          }
+              @Override
+              public ScanResultValue next()
+              {
+                if (!hasNext()) {
+                  throw new NoSuchElementException();
+                }
+                if (hasTimeout && System.currentTimeMillis() >= timeoutAt) {
+                  throw new QueryTimeoutException(StringUtils.nonStrictFormat("Query [%s] timed out", query.getId()));
+                }
+                final long lastOffset = offset;
+                final Object events;
+                final ScanQuery.ResultFormat resultFormat = query.getResultFormat();
+                if (ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST.equals(resultFormat)) {
+                  events = rowsToCompactedList();
+                } else if (ScanQuery.ResultFormat.RESULT_FORMAT_LIST.equals(resultFormat)) {
+                  events = rowsToList();
+                } else {
+                  throw new UOE("resultFormat[%s] is not supported", resultFormat.toString());
+                }
+                responseContext.addRowScanCount(offset - lastOffset);
+                return new ScanResultValue(segmentId.toString(), allColumns, events, rowSignatureBuilder.build());
+              }
 
-                          @Override
-                          public void remove()
-                          {
-                            throw new UnsupportedOperationException();
-                          }
+              @Override
+              public void remove()
+              {
+                throw new UnsupportedOperationException();
+              }
 
-                          private List<List<Object>> rowsToCompactedList()
-                          {
-                            final List<List<Object>> events = new ArrayList<>(batchSize);
-                            final long iterLimit = Math.min(limit, offset + batchSize);
-                            for (; !cursor.isDone() && offset < iterLimit; cursor.advance(), offset++) {
-                              final List<Object> theEvent = new ArrayList<>(allColumns.size());
-                              for (int j = 0; j < allColumns.size(); j++) {
-                                theEvent.add(getColumnValue(j));
-                              }
-                              events.add(theEvent);
-                            }
-                            return events;
-                          }
+              private List<List<Object>> rowsToCompactedList()
+              {
+                final List<List<Object>> events = new ArrayList<>(batchSize);
+                final long iterLimit = Math.min(limit, offset + batchSize);
+                for (; !cursor.isDone() && offset < iterLimit; cursor.advance(), offset++) {
+                  final List<Object> theEvent = new ArrayList<>(allColumns.size());
+                  for (int j = 0; j < allColumns.size(); j++) {
+                    theEvent.add(getColumnValue(j));
+                  }
+                  events.add(theEvent);
+                }
+                return events;
+              }
 
-                          private List<Map<String, Object>> rowsToList()
-                          {
-                            List<Map<String, Object>> events = Lists.newArrayListWithCapacity(batchSize);
-                            final long iterLimit = Math.min(limit, offset + batchSize);
-                            for (; !cursor.isDone() && offset < iterLimit; cursor.advance(), offset++) {
-                              final Map<String, Object> theEvent = new LinkedHashMap<>();
-                              for (int j = 0; j < allColumns.size(); j++) {
-                                theEvent.put(allColumns.get(j), getColumnValue(j));
-                              }
-                              events.add(theEvent);
-                            }
-                            return events;
-                          }
+              private List<Map<String, Object>> rowsToList()
+              {
+                List<Map<String, Object>> events = Lists.newArrayListWithCapacity(batchSize);
+                final long iterLimit = Math.min(limit, offset + batchSize);
+                for (; !cursor.isDone() && offset < iterLimit; cursor.advance(), offset++) {
+                  final Map<String, Object> theEvent = new LinkedHashMap<>();
+                  for (int j = 0; j < allColumns.size(); j++) {
+                    theEvent.put(allColumns.get(j), getColumnValue(j));
+                  }
+                  events.add(theEvent);
+                }
+                return events;
+              }
 
-                          private Object getColumnValue(int i)
-                          {
-                            final BaseObjectColumnValueSelector selector = columnSelectors.get(i);
-                            final Object value = selector == null ? null : selector.getObject();
-                            return value;
-                          }
-                        };
-                      }
+              private Object getColumnValue(int i)
+              {
+                final BaseObjectColumnValueSelector selector = columnSelectors.get(i);
+                final Object value = selector == null ? null : selector.getObject();
+                return value;
+              }
+            };
+          }
 
-                      @Override
-                      public void cleanup(Iterator<ScanResultValue> iterFromMake)
-                      {
-                      }
-                    }
-            ))
-    );
+          @Override
+          public void cleanup(Iterator<ScanResultValue> iterFromMake)
+          {
+          }
+        }
+    ).withBaggage(cursorHolder);
   }
 
   /**
@@ -241,9 +238,21 @@ public class ScanQueryEngine
    */
   private long calculateRemainingScanRowsLimit(ScanQuery query, ResponseContext responseContext)
   {
-    if (query.getTimeOrder().equals(ScanQuery.Order.NONE)) {
+    if (query.getTimeOrder().equals(Order.NONE)) {
       return query.getScanRowsLimit() - (Long) responseContext.getRowScanCount();
     }
     return query.getScanRowsLimit();
   }
+
+  public static CursorBuildSpec makeCursorBuildSpec(ScanQuery query, @Nullable QueryMetrics<?> queryMetrics)
+  {
+    return CursorBuildSpec.builder()
+                          .setInterval(query.getSingleInterval())
+                          .setFilter(Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter())))
+                          .setVirtualColumns(query.getVirtualColumns())
+                          .setPreferredOrdering(query.getOrderBys())
+                          .setQueryContext(query.context())
+                          .setQueryMetrics(queryMetrics)
+                          .build();
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryLimitRowIterator.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryLimitRowIterator.java
index 785b71ed42d..45cd4c2624c 100644
--- a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryLimitRowIterator.java
+++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryLimitRowIterator.java
@@ -26,6 +26,7 @@ import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Yielder;
 import org.apache.druid.java.util.common.guava.YieldingAccumulator;
 import org.apache.druid.java.util.common.parsers.CloseableIterator;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryRunner;
@@ -99,7 +100,7 @@ public class ScanQueryLimitRowIterator implements CloseableIterator<ScanResultVa
 
     // We want to perform multi-event ScanResultValue limiting if we are not time-ordering or are at the
     // inner-level if we are time-ordering
-    if (query.getTimeOrder() == ScanQuery.Order.NONE ||
+    if (query.getTimeOrder() == Order.NONE ||
         !query.context().getBoolean(ScanQuery.CTX_KEY_OUTERMOST, true)) {
       ScanResultValue batch = yielder.get();
       List events = (List) batch.getEvents();
diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryRunnerFactory.java
index 0013c4f84fc..f09b42c2795 100644
--- a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryRunnerFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryRunnerFactory.java
@@ -32,6 +32,7 @@ import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.guava.Yielder;
 import org.apache.druid.java.util.common.guava.Yielders;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryProcessingPool;
@@ -96,7 +97,7 @@ public class ScanQueryRunnerFactory implements QueryRunnerFactory<ScanResultValu
       final long timeoutAt = System.currentTimeMillis() + queryPlus.getQuery().context().getTimeout();
       responseContext.putTimeoutTime(timeoutAt);
 
-      if (query.getTimeOrder().equals(ScanQuery.Order.NONE)) {
+      if (query.getTimeOrder().equals(Order.NONE)) {
         // Use normal strategy
         Sequence<ScanResultValue> returnedRows = Sequences.concat(
             Sequences.map(
@@ -113,7 +114,7 @@ public class ScanQueryRunnerFactory implements QueryRunnerFactory<ScanResultValu
         List<Interval> intervalsOrdered = getIntervalsFromSpecificQuerySpec(query.getQuerySegmentSpec());
         List<QueryRunner<ScanResultValue>> queryRunnersOrdered = Lists.newArrayList(queryRunners);
 
-        if (ScanQuery.Order.DESCENDING.equals(query.getTimeOrder())) {
+        if (Order.DESCENDING.equals(query.getTimeOrder())) {
           intervalsOrdered = Lists.reverse(intervalsOrdered);
           queryRunnersOrdered = Lists.reverse(queryRunnersOrdered);
         }
diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanResultValueTimestampComparator.java b/processing/src/main/java/org/apache/druid/query/scan/ScanResultValueTimestampComparator.java
index 9724c2408ba..e4c353a8185 100644
--- a/processing/src/main/java/org/apache/druid/query/scan/ScanResultValueTimestampComparator.java
+++ b/processing/src/main/java/org/apache/druid/query/scan/ScanResultValueTimestampComparator.java
@@ -20,6 +20,7 @@
 package org.apache.druid.query.scan;
 
 import com.google.common.primitives.Longs;
+import org.apache.druid.query.Order;
 
 import java.util.Comparator;
 
@@ -45,7 +46,7 @@ public class ScanResultValueTimestampComparator implements Comparator<ScanResult
     int comparison = Longs.compare(
         o1.getFirstEventTimestamp(scanQuery.getResultFormat()),
         o2.getFirstEventTimestamp(scanQuery.getResultFormat()));
-    if (scanQuery.getTimeOrder().equals(ScanQuery.Order.ASCENDING)) {
+    if (scanQuery.getTimeOrder().equals(Order.ASCENDING)) {
       return comparison;
     }
     return comparison * -1;
diff --git a/processing/src/main/java/org/apache/druid/query/search/CursorOnlyStrategy.java b/processing/src/main/java/org/apache/druid/query/search/CursorOnlyStrategy.java
index 77325c58233..b6d0cb5f11a 100644
--- a/processing/src/main/java/org/apache/druid/query/search/CursorOnlyStrategy.java
+++ b/processing/src/main/java/org/apache/druid/query/search/CursorOnlyStrategy.java
@@ -21,16 +21,16 @@ package org.apache.druid.query.search;
 
 import com.google.common.collect.ImmutableList;
 import it.unimi.dsi.fastutil.objects.Object2IntRBTreeMap;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.ColumnSelectorPlus;
 import org.apache.druid.query.dimension.DimensionSpec;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.search.SearchQueryRunner.SearchColumnSelectorStrategy;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionHandlerUtils;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.StorageAdapter;
-import org.joda.time.Interval;
+import org.apache.druid.segment.filter.Filters;
 
 import java.util.List;
 
@@ -56,84 +56,72 @@ public class CursorOnlyStrategy extends SearchStrategy
     return ImmutableList.of(new CursorBasedExecutor(
         query,
         segment,
-        filter,
-        interval,
         dimensionSpecs
     ));
   }
 
   public static class CursorBasedExecutor extends SearchQueryExecutor
   {
-
-    protected Filter filter;
-    protected Interval interval;
-
     public CursorBasedExecutor(
         SearchQuery query,
         Segment segment,
-        Filter filter,
-        Interval interval, List<DimensionSpec> dimensionSpecs
+        List<DimensionSpec> dimensionSpecs
     )
     {
       super(query, segment, dimensionSpecs);
-
-      this.filter = filter;
-      this.interval = interval;
     }
 
     @Override
     public Object2IntRBTreeMap<SearchHit> execute(final int limit)
     {
       final StorageAdapter adapter = segment.asStorageAdapter();
+      final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                       .setInterval(query.getSingleInterval())
+                                                       .setFilter(
+                                                           Filters.convertToCNFFromQueryContext(
+                                                               query,
+                                                               Filters.toFilter(query.getFilter())
+                                                           )
+                                                       )
+                                                       .setVirtualColumns(query.getVirtualColumns())
+                                                       .setQueryContext(query.context())
+                                                       .build();
+      try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+        final Cursor cursor = cursorHolder.asCursor();
 
-      final Sequence<Cursor> cursors = adapter.makeCursors(
-          filter,
-          interval,
-          query.getVirtualColumns(),
-          query.getGranularity(),
-          query.isDescending(),
-          null
-      );
+        final Object2IntRBTreeMap<SearchHit> retVal = new Object2IntRBTreeMap<>(query.getSort().getComparator());
+        retVal.defaultReturnValue(0);
 
-      final Object2IntRBTreeMap<SearchHit> retVal = new Object2IntRBTreeMap<>(query.getSort().getComparator());
-      retVal.defaultReturnValue(0);
+        if (cursor == null) {
+          return retVal;
+        }
 
-      cursors.accumulate(
-          retVal,
-          (map, cursor) -> {
-            if (map.size() >= limit) {
-              return map;
-            }
+        final ColumnSelectorPlus<SearchColumnSelectorStrategy>[] selectorPlusList = DimensionHandlerUtils.createColumnSelectorPluses(
+            SearchQueryRunner.SEARCH_COLUMN_SELECTOR_STRATEGY_FACTORY,
+            dimsToSearch,
+            cursor.getColumnSelectorFactory()
+        );
 
-            final ColumnSelectorPlus<SearchColumnSelectorStrategy>[] selectorPlusList = DimensionHandlerUtils.createColumnSelectorPluses(
-                SearchQueryRunner.SEARCH_COLUMN_SELECTOR_STRATEGY_FACTORY,
-                dimsToSearch,
-                cursor.getColumnSelectorFactory()
+        while (!cursor.isDone()) {
+          for (ColumnSelectorPlus<SearchColumnSelectorStrategy> selectorPlus : selectorPlusList) {
+            selectorPlus.getColumnSelectorStrategy().updateSearchResultSet(
+                selectorPlus.getOutputName(),
+                selectorPlus.getSelector(),
+                searchQuerySpec,
+                limit,
+                retVal
             );
 
-            while (!cursor.isDone()) {
-              for (ColumnSelectorPlus<SearchColumnSelectorStrategy> selectorPlus : selectorPlusList) {
-                selectorPlus.getColumnSelectorStrategy().updateSearchResultSet(
-                    selectorPlus.getOutputName(),
-                    selectorPlus.getSelector(),
-                    searchQuerySpec,
-                    limit,
-                    map
-                );
-
-                if (map.size() >= limit) {
-                  return map;
-                }
-              }
-
-              cursor.advance();
+            if (retVal.size() >= limit) {
+              return retVal;
             }
-
-            return map;
           }
-      );
 
-      return retVal;
+          cursor.advance();
+        }
+
+        return retVal;
+      }
     }
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/search/SearchQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/search/SearchQueryQueryToolChest.java
index c15e1d0d99c..3519de9b2f8 100644
--- a/processing/src/main/java/org/apache/druid/query/search/SearchQueryQueryToolChest.java
+++ b/processing/src/main/java/org/apache/druid/query/search/SearchQueryQueryToolChest.java
@@ -97,7 +97,7 @@ public class SearchQueryQueryToolChest extends QueryToolChest<Result<SearchResul
   @Override
   public Comparator<Result<SearchResultValue>> createResultComparator(Query<Result<SearchResultValue>> query)
   {
-    return ResultGranularTimestampComparator.create(query.getGranularity(), query.isDescending());
+    return ResultGranularTimestampComparator.create(query.getGranularity(), false);
   }
 
   @Override
diff --git a/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
index 8fc431a756b..c522e70c90b 100644
--- a/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
+++ b/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
@@ -114,10 +114,10 @@ public class UseIndexesStrategy extends SearchStrategy
       }
 
       if (nonBitmapSuppDims.size() > 0) {
-        builder.add(new CursorBasedExecutor(query, segment, filter, interval, nonBitmapSuppDims));
+        builder.add(new CursorBasedExecutor(query, segment, nonBitmapSuppDims));
       }
     } else {
-      builder.add(new CursorBasedExecutor(query, segment, filter, interval, searchDims));
+      builder.add(new CursorBasedExecutor(query, segment, searchDims));
     }
 
     return builder.build();
diff --git a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java
index aac161c2728..17db30455e0 100644
--- a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java
@@ -66,7 +66,7 @@ public class TimeBoundaryQuery extends BaseQuery<Result<TimeBoundaryResultValue>
       @JsonProperty("context") Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec == null ? DEFAULT_SEGMENT_SPEC : querySegmentSpec, false, context);
+    super(dataSource, querySegmentSpec == null ? DEFAULT_SEGMENT_SPEC : querySegmentSpec, context);
 
     this.dimFilter = dimFilter;
     this.bound = bound == null ? "" : bound;
diff --git a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
index 7a695987fb7..38b5ff6eb93 100644
--- a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
@@ -24,16 +24,15 @@ import com.google.inject.Inject;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.BaseSequence;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.ChainedExecutionQueryRunner;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryProcessingPool;
 import org.apache.druid.query.QueryRunner;
 import org.apache.druid.query.QueryRunnerFactory;
-import org.apache.druid.query.QueryRunnerHelper;
 import org.apache.druid.query.QueryToolChest;
 import org.apache.druid.query.QueryWatcher;
 import org.apache.druid.query.Result;
@@ -41,17 +40,19 @@ import org.apache.druid.query.TableDataSource;
 import org.apache.druid.query.context.ResponseContext;
 import org.apache.druid.segment.BaseLongColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.utils.CollectionUtils;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
+import javax.annotation.Nullable;
 import java.util.Iterator;
-import java.util.List;
 
 /**
  *
@@ -114,24 +115,25 @@ public class TimeBoundaryQueryRunnerFactory
       };
     }
 
+    @Nullable
     private DateTime getTimeBoundary(StorageAdapter adapter, TimeBoundaryQuery legacyQuery, boolean descending)
     {
-      final Sequence<Result<DateTime>> resultSequence = QueryRunnerHelper.makeCursorBasedQuery(
-          adapter,
-          legacyQuery.getQuerySegmentSpec().getIntervals(),
-          Filters.toFilter(legacyQuery.getFilter()),
-          VirtualColumns.EMPTY,
-          descending,
-          Granularities.ALL,
-          this.skipToFirstMatching,
-          null
-      );
-      final List<Result<DateTime>> resultList = resultSequence.limit(1).toList();
-      if (resultList.size() > 0) {
-        return resultList.get(0).getValue();
+      final CursorBuildSpec.CursorBuildSpecBuilder bob = CursorBuildSpec.builder(makeCursorBuildSpec(legacyQuery));
+      if (descending) {
+        bob.setPreferredOrdering(Cursors.descendingTimeOrder());
+      } else {
+        bob.setPreferredOrdering(Cursors.ascendingTimeOrder());
       }
 
-      return null;
+      try (final CursorHolder cursorHolder = adapter.makeCursorHolder(bob.build())) {
+        Cursors.requireTimeOrdering(cursorHolder, descending ? Order.DESCENDING : Order.ASCENDING);
+        final Cursor cursor = cursorHolder.asCursor();
+        if (cursor == null) {
+          return null;
+        }
+        final Result<DateTime> result = skipToFirstMatching.apply(cursor);
+        return result == null ? null : result.getValue();
+      }
     }
 
     @Override
@@ -201,7 +203,7 @@ public class TimeBoundaryQueryRunnerFactory
 
   /**
    * Whether a particular {@link TimeBoundaryQuery} can use {@link StorageAdapter#getMinTime()} and/or
-   * {@link StorageAdapter#getMaxTime()}. If false, must use {@link StorageAdapter#makeCursors}.
+   * {@link StorageAdapter#getMaxTime()}. If false, must use {@link StorageAdapter#makeCursorHolder(CursorBuildSpec)}.
    */
   private static boolean canUseAdapterMinMaxTime(final TimeBoundaryQuery query, final StorageAdapter adapter)
   {
@@ -232,4 +234,14 @@ public class TimeBoundaryQueryRunnerFactory
     // Passed all checks.
     return true;
   }
+
+  public static CursorBuildSpec makeCursorBuildSpec(TimeBoundaryQuery query)
+  {
+    return CursorBuildSpec.builder()
+                          .setInterval(query.getSingleInterval())
+                          .setFilter(Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter())))
+                          .setVirtualColumns(query.getVirtualColumns())
+                          .setQueryContext(query.context())
+                          .build();
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java
index d255a4e4cf4..85ddf5aaa62 100644
--- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java
@@ -191,8 +191,7 @@ public class TimeseriesQuery extends BaseQuery<Result<TimeseriesResultValue>>
         virtualColumns,
         dimFilter,
         Collections.emptyList(),
-        aggregatorSpecs,
-        Collections.emptyList()
+        aggregatorSpecs
     );
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryEngine.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryEngine.java
index d8369c8c6da..e23f8e19e6d 100644
--- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryEngine.java
@@ -27,22 +27,26 @@ import org.apache.druid.collections.ResourceHolder;
 import org.apache.druid.collections.StupidPool;
 import org.apache.druid.guice.annotations.Global;
 import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.query.QueryRunnerHelper;
+import org.apache.druid.query.CursorGranularizer;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.query.aggregation.AggregatorAdapters;
 import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.query.filter.Filter;
-import org.apache.druid.query.groupby.epinephelinae.vector.VectorGroupByEngine;
 import org.apache.druid.query.vector.VectorCursorGranularizer;
-import org.apache.druid.segment.ColumnInspector;
+import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.SegmentMissingException;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
 import org.apache.druid.segment.vector.VectorCursor;
@@ -50,7 +54,6 @@ import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
 import java.nio.ByteBuffer;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -94,64 +97,52 @@ public class TimeseriesQueryEngine
       );
     }
 
-    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter()));
     final Interval interval = Iterables.getOnlyElement(query.getIntervals());
     final Granularity gran = query.getGranularity();
-    final boolean descending = query.isDescending();
 
-    final ColumnInspector inspector = query.getVirtualColumns().wrapInspector(adapter);
 
-    final boolean doVectorize = query.context().getVectorize().shouldVectorize(
-        adapter.canVectorize(filter, query.getVirtualColumns(), descending)
-        && VirtualColumns.shouldVectorize(query, query.getVirtualColumns(), adapter)
-        && VectorGroupByEngine.canVectorizeAggregators(inspector, query.getAggregatorSpecs())
-    );
+    final CursorHolder cursorHolder = adapter.makeCursorHolder(makeCursorBuildSpec(query, timeseriesQueryMetrics));
+    Cursors.requireTimeOrdering(cursorHolder, query.isDescending() ? Order.DESCENDING : Order.ASCENDING);
+    try {
+      final Sequence<Result<TimeseriesResultValue>> result;
 
-    final Sequence<Result<TimeseriesResultValue>> result;
+      if (query.context().getVectorize().shouldVectorize(cursorHolder.canVectorize())) {
+        result = processVectorized(query, adapter, cursorHolder, interval, gran);
+      } else {
+        result = processNonVectorized(query, adapter, cursorHolder, interval, gran);
+      }
 
-    if (doVectorize) {
-      result = processVectorized(query, adapter, filter, interval, gran, descending, timeseriesQueryMetrics);
-    } else {
-      result = processNonVectorized(query, adapter, filter, interval, gran, descending, timeseriesQueryMetrics);
+      final int limit = query.getLimit();
+      if (limit < Integer.MAX_VALUE) {
+        return result.limit(limit).withBaggage(cursorHolder);
+      } else {
+        return result.withBaggage(cursorHolder);
+      }
     }
-
-    final int limit = query.getLimit();
-    if (limit < Integer.MAX_VALUE) {
-      return result.limit(limit);
-    } else {
-      return result;
+    catch (Throwable t) {
+      cursorHolder.close();
+      throw t;
     }
   }
 
   private Sequence<Result<TimeseriesResultValue>> processVectorized(
       final TimeseriesQuery query,
       final StorageAdapter adapter,
-      @Nullable final Filter filter,
+      final CursorHolder cursorHolder,
       final Interval queryInterval,
-      final Granularity gran,
-      final boolean descending,
-      final TimeseriesQueryMetrics timeseriesQueryMetrics
+      final Granularity gran
   )
   {
     final boolean skipEmptyBuckets = query.isSkipEmptyBuckets();
     final List<AggregatorFactory> aggregatorSpecs = query.getAggregatorSpecs();
 
-    final VectorCursor cursor = adapter.makeVectorCursor(
-        filter,
-        queryInterval,
-        query.getVirtualColumns(),
-        descending,
-        query.context().getVectorSize(),
-        timeseriesQueryMetrics
-    );
+    final VectorCursor cursor = cursorHolder.asVectorCursor();
 
     if (cursor == null) {
       return Sequences.empty();
     }
 
     final Closer closer = Closer.create();
-    closer.register(cursor);
-
     try {
       final VectorCursorGranularizer granularizer = VectorCursorGranularizer.create(
           adapter,
@@ -253,60 +244,98 @@ public class TimeseriesQueryEngine
   private Sequence<Result<TimeseriesResultValue>> processNonVectorized(
       final TimeseriesQuery query,
       final StorageAdapter adapter,
-      @Nullable final Filter filter,
+      final CursorHolder cursorHolder,
       final Interval queryInterval,
-      final Granularity gran,
-      final boolean descending,
-      final TimeseriesQueryMetrics timeseriesQueryMetrics
+      final Granularity gran
   )
   {
     final boolean skipEmptyBuckets = query.isSkipEmptyBuckets();
     final List<AggregatorFactory> aggregatorSpecs = query.getAggregatorSpecs();
-
-    return QueryRunnerHelper.makeCursorBasedQuery(
+    final Cursor cursor = cursorHolder.asCursor();
+    if (cursor == null) {
+      return Sequences.empty();
+    }
+    final CursorGranularizer granularizer = CursorGranularizer.create(
         adapter,
-        Collections.singletonList(queryInterval),
-        filter,
-        query.getVirtualColumns(),
-        descending,
+        cursor,
         gran,
-        cursor -> {
-          if (skipEmptyBuckets && cursor.isDone()) {
-            return null;
-          }
+        queryInterval,
+        query.isDescending()
+    );
+    if (granularizer == null) {
+      return Sequences.empty();
+    }
+    final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+    return Sequences.simple(granularizer.getBucketIterable())
+                    .map(
+                        bucketInterval -> {
+                          // Whether or not the current bucket is empty
+                          boolean emptyBucket = true;
+                          boolean advancedToBucket = granularizer.advanceToBucket(bucketInterval);
+                          if ((!advancedToBucket || cursor.isDone()) && skipEmptyBuckets) {
+                            return null;
+                          }
+                          final Aggregator[] aggregators = new Aggregator[aggregatorSpecs.size()];
+                          final String[] aggregatorNames = new String[aggregatorSpecs.size()];
 
-          Aggregator[] aggregators = new Aggregator[aggregatorSpecs.size()];
-          String[] aggregatorNames = new String[aggregatorSpecs.size()];
+                          for (int i = 0; i < aggregatorSpecs.size(); i++) {
+                            aggregators[i] = aggregatorSpecs.get(i).factorize(columnSelectorFactory);
+                            aggregatorNames[i] = aggregatorSpecs.get(i).getName();
+                          }
+                          try {
+                            if (advancedToBucket) {
+                              while (!cursor.isDone()) {
+                                for (Aggregator aggregator : aggregators) {
+                                  aggregator.aggregate();
+                                }
+                                emptyBucket = false;
 
-          for (int i = 0; i < aggregatorSpecs.size(); i++) {
-            aggregators[i] = aggregatorSpecs.get(i).factorize(cursor.getColumnSelectorFactory());
-            aggregatorNames[i] = aggregatorSpecs.get(i).getName();
-          }
+                                if (!granularizer.advanceCursorWithinBucket()) {
+                                  break;
+                                }
+                              }
+                            }
 
-          try {
-            while (!cursor.isDone()) {
-              for (Aggregator aggregator : aggregators) {
-                aggregator.aggregate();
-              }
-              cursor.advance();
-            }
+                            if (emptyBucket && skipEmptyBuckets) {
+                              // Return null, will get filtered out later by the Objects::nonNull filter.
+                              return null;
+                            }
 
-            TimeseriesResultBuilder bob = new TimeseriesResultBuilder(cursor.getTime());
+                            final TimeseriesResultBuilder bob = new TimeseriesResultBuilder(
+                                gran.toDateTime(bucketInterval.getStartMillis())
+                            );
+                            for (int i = 0; i < aggregatorSpecs.size(); i++) {
+                              bob.addMetric(aggregatorNames[i], aggregators[i].get());
+                            }
 
-            for (int i = 0; i < aggregatorSpecs.size(); i++) {
-              bob.addMetric(aggregatorNames[i], aggregators[i].get());
-            }
+                            return bob.build();
+                          }
+                          finally {
+                            // cleanup
+                            for (Aggregator agg : aggregators) {
+                              agg.close();
+                            }
+                          }
+                        }
+                    )
+                    .filter(Objects::nonNull);
+  }
 
-            return bob.build();
-          }
-          finally {
-            // cleanup
-            for (Aggregator agg : aggregators) {
-              agg.close();
-            }
-          }
-        },
-        timeseriesQueryMetrics
+  public static CursorBuildSpec makeCursorBuildSpec(TimeseriesQuery query, @Nullable QueryMetrics<?> queryMetrics)
+  {
+    return Granularities.decorateCursorBuildSpec(
+        query,
+        CursorBuildSpec.builder()
+                       .setInterval(query.getSingleInterval())
+                       .setFilter(Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter())))
+                       .setVirtualColumns(query.getVirtualColumns())
+                       .setAggregators(query.getAggregatorSpecs())
+                       .setQueryContext(query.context())
+                       .setPreferredOrdering(
+                           query.isDescending() ? Cursors.descendingTimeOrder() : Cursors.ascendingTimeOrder()
+                       )
+                       .setQueryMetrics(queryMetrics)
+                       .build()
     );
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/topn/AggregateTopNMetricFirstAlgorithm.java b/processing/src/main/java/org/apache/druid/query/topn/AggregateTopNMetricFirstAlgorithm.java
index 2180b73253c..99759f515dd 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/AggregateTopNMetricFirstAlgorithm.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/AggregateTopNMetricFirstAlgorithm.java
@@ -23,6 +23,7 @@ import org.apache.druid.collections.NonBlockingPool;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.query.ColumnSelectorPlus;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.AggregatorUtil;
 import org.apache.druid.query.aggregation.PostAggregator;
@@ -62,9 +63,9 @@ public class AggregateTopNMetricFirstAlgorithm implements TopNAlgorithm<int[], T
   }
 
   @Override
-  public TopNParams makeInitParams(ColumnSelectorPlus selectorPlus, Cursor cursor)
+  public TopNParams makeInitParams(ColumnSelectorPlus selectorPlus, Cursor cursor, CursorGranularizer granularizer)
   {
-    return new TopNParams(selectorPlus, cursor, Integer.MAX_VALUE);
+    return new TopNParams(selectorPlus, cursor, granularizer, Integer.MAX_VALUE);
   }
 
   @Override
@@ -93,7 +94,7 @@ public class AggregateTopNMetricFirstAlgorithm implements TopNAlgorithm<int[], T
     PooledTopNAlgorithm.PooledTopNParams singleMetricParam = null;
     int[] dimValSelector;
     try {
-      singleMetricParam = singleMetricAlgo.makeInitParams(params.getSelectorPlus(), params.getCursor());
+      singleMetricParam = singleMetricAlgo.makeInitParams(params.getSelectorPlus(), params.getCursor(), params.getGranularizer());
       singleMetricAlgo.run(
           singleMetricParam,
           singleMetricResultBuilder,
@@ -111,8 +112,10 @@ public class AggregateTopNMetricFirstAlgorithm implements TopNAlgorithm<int[], T
     PooledTopNAlgorithm allMetricAlgo = new PooledTopNAlgorithm(storageAdapter, query, bufferPool);
     PooledTopNAlgorithm.PooledTopNParams allMetricsParam = null;
     try {
+      // reset cursor since we call run again
+      params.getCursor().reset();
       // Run topN for all metrics for top N dimension values
-      allMetricsParam = allMetricAlgo.makeInitParams(params.getSelectorPlus(), params.getCursor());
+      allMetricsParam = allMetricAlgo.makeInitParams(params.getSelectorPlus(), params.getCursor(), params.getGranularizer());
       allMetricAlgo.run(
           allMetricsParam,
           resultBuilder,
diff --git a/processing/src/main/java/org/apache/druid/query/topn/BaseTopNAlgorithm.java b/processing/src/main/java/org/apache/druid/query/topn/BaseTopNAlgorithm.java
index f34464a49d0..c8bcbc1a094 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/BaseTopNAlgorithm.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/BaseTopNAlgorithm.java
@@ -123,7 +123,9 @@ public abstract class BaseTopNAlgorithm<DimValSelector, DimValAggregateStore, Pa
       resetAggregators(aggregatesStore);
 
       numProcessed += numToProcess;
-      params.getCursor().reset();
+      if (numProcessed < cardinality) {
+        params.getCursor().reset();
+      }
     }
     if (queryMetrics != null) {
       queryMetrics.addProcessedRows(processedRows);
@@ -152,7 +154,6 @@ public abstract class BaseTopNAlgorithm<DimValSelector, DimValAggregateStore, Pa
     long processedRows = scanAndAggregate(params, null, aggregatesStore);
     updateResults(params, null, aggregatesStore, resultBuilder);
     resetAggregators(aggregatesStore);
-    params.getCursor().reset();
     if (queryMetrics != null) {
       queryMetrics.addProcessedRows(processedRows);
       queryMetrics.stopRecordingScanTime();
@@ -330,7 +331,7 @@ public abstract class BaseTopNAlgorithm<DimValSelector, DimValAggregateStore, Pa
     final Comparator comparator = query.getTopNMetricSpec()
                                        .getComparator(query.getAggregatorSpecs(), query.getPostAggregatorSpecs());
     return query.getTopNMetricSpec().getResultBuilder(
-        params.getCursor().getTime(),
+        params.getGranularizer().getBucketStart(),
         query.getDimensionSpec(),
         query.getThreshold(),
         comparator,
diff --git a/processing/src/main/java/org/apache/druid/query/topn/Generic1AggPooledTopNScanner.java b/processing/src/main/java/org/apache/druid/query/topn/Generic1AggPooledTopNScanner.java
index 544cef0dfaf..c4d653ec044 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/Generic1AggPooledTopNScanner.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/Generic1AggPooledTopNScanner.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.query.topn;
 
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.BufferAggregator;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.DimensionSelector;
@@ -37,6 +38,7 @@ public interface Generic1AggPooledTopNScanner
       BufferAggregator aggregator,
       int aggregatorSize,
       Cursor cursor,
+      CursorGranularizer granularizer,
       int[] positions,
       ByteBuffer resultsBuffer
   );
diff --git a/processing/src/main/java/org/apache/druid/query/topn/Generic1AggPooledTopNScannerPrototype.java b/processing/src/main/java/org/apache/druid/query/topn/Generic1AggPooledTopNScannerPrototype.java
index 3200c625b2f..f4c2ba1863b 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/Generic1AggPooledTopNScannerPrototype.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/Generic1AggPooledTopNScannerPrototype.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.query.topn;
 
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.BufferAggregator;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.DimensionSelector;
@@ -46,6 +47,7 @@ public final class Generic1AggPooledTopNScannerPrototype implements Generic1AggP
       BufferAggregator aggregator,
       int aggregatorSize,
       Cursor cursor,
+      CursorGranularizer granularizer,
       int[] positions,
       ByteBuffer resultsBuffer
   )
@@ -69,7 +71,9 @@ public final class Generic1AggPooledTopNScannerPrototype implements Generic1AggP
         }
       }
       processedRows++;
-      cursor.advanceUninterruptibly();
+      if (!granularizer.advanceCursorWithinBucketUninterruptedly()) {
+        break;
+      }
     }
     return processedRows;
   }
diff --git a/processing/src/main/java/org/apache/druid/query/topn/Generic2AggPooledTopNScanner.java b/processing/src/main/java/org/apache/druid/query/topn/Generic2AggPooledTopNScanner.java
index 6b6adb881c0..7f74b4b7c39 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/Generic2AggPooledTopNScanner.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/Generic2AggPooledTopNScanner.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.query.topn;
 
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.BufferAggregator;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.DimensionSelector;
@@ -40,6 +41,7 @@ public interface Generic2AggPooledTopNScanner
       BufferAggregator aggregator2,
       int aggregator2Size,
       Cursor cursor,
+      CursorGranularizer granularizer,
       int[] positions,
       ByteBuffer resultsBuffer
   );
diff --git a/processing/src/main/java/org/apache/druid/query/topn/Generic2AggPooledTopNScannerPrototype.java b/processing/src/main/java/org/apache/druid/query/topn/Generic2AggPooledTopNScannerPrototype.java
index d7536cb087f..4de281d8a0b 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/Generic2AggPooledTopNScannerPrototype.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/Generic2AggPooledTopNScannerPrototype.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.query.topn;
 
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.BufferAggregator;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.DimensionSelector;
@@ -48,6 +49,7 @@ public final class Generic2AggPooledTopNScannerPrototype implements Generic2AggP
       BufferAggregator aggregator2,
       int aggregator2Size,
       Cursor cursor,
+      CursorGranularizer granularizer,
       int[] positions,
       ByteBuffer resultsBuffer
   )
@@ -76,7 +78,9 @@ public final class Generic2AggPooledTopNScannerPrototype implements Generic2AggP
         }
       }
       processedRows++;
-      cursor.advanceUninterruptibly();
+      if (!granularizer.advanceCursorWithinBucketUninterruptedly()) {
+        break;
+      }
     }
     return processedRows;
   }
diff --git a/processing/src/main/java/org/apache/druid/query/topn/HeapBasedTopNAlgorithm.java b/processing/src/main/java/org/apache/druid/query/topn/HeapBasedTopNAlgorithm.java
index ba5fbf25108..6f8ed23bc29 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/HeapBasedTopNAlgorithm.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/HeapBasedTopNAlgorithm.java
@@ -20,6 +20,7 @@
 package org.apache.druid.query.topn;
 
 import org.apache.druid.query.ColumnSelectorPlus;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.query.topn.types.TopNColumnAggregatesProcessor;
 import org.apache.druid.segment.Cursor;
@@ -48,12 +49,14 @@ public class HeapBasedTopNAlgorithm
   @Override
   public TopNParams makeInitParams(
       final ColumnSelectorPlus<TopNColumnAggregatesProcessor> selectorPlus,
-      final Cursor cursor
+      final Cursor cursor,
+      final CursorGranularizer granularizer
   )
   {
     return new TopNParams(
         selectorPlus,
         cursor,
+        granularizer,
         Integer.MAX_VALUE
     );
   }
@@ -96,6 +99,7 @@ public class HeapBasedTopNAlgorithm
         query,
         selectorPlus.getSelector(),
         cursor,
+        params.getGranularizer(),
         rowSelector
     );
   }
diff --git a/processing/src/main/java/org/apache/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/org/apache/druid/query/topn/PooledTopNAlgorithm.java
index d0c0fb064e0..d9213b94e09 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/PooledTopNAlgorithm.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/PooledTopNAlgorithm.java
@@ -26,6 +26,7 @@ import org.apache.druid.collections.ResourceHolder;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.ColumnSelectorPlus;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.BufferAggregator;
 import org.apache.druid.query.aggregation.SimpleDoubleBufferAggregator;
 import org.apache.druid.query.monomorphicprocessing.SpecializationService;
@@ -191,7 +192,7 @@ public class PooledTopNAlgorithm
     if (SPECIALIZE_GENERIC_ONE_AGG_POOLED_TOPN) {
       SPECIALIZED_SCAN_AND_AGGREGATE_IMPLEMENTATIONS.add((params, positions, theAggregators) -> {
         if (theAggregators.length == 1) {
-          return scanAndAggregateGeneric1Agg(params, positions, theAggregators[0], params.getCursor());
+          return scanAndAggregateGeneric1Agg(params, positions, theAggregators[0]);
         }
         return -1;
       });
@@ -199,7 +200,7 @@ public class PooledTopNAlgorithm
     if (SPECIALIZE_GENERIC_TWO_AGG_POOLED_TOPN) {
       SPECIALIZED_SCAN_AND_AGGREGATE_IMPLEMENTATIONS.add((params, positions, theAggregators) -> {
         if (theAggregators.length == 2) {
-          return scanAndAggregateGeneric2Agg(params, positions, theAggregators, params.getCursor());
+          return scanAndAggregateGeneric2Agg(params, positions, theAggregators);
         }
         return -1;
       });
@@ -222,7 +223,7 @@ public class PooledTopNAlgorithm
   }
 
   @Override
-  public PooledTopNParams makeInitParams(ColumnSelectorPlus selectorPlus, Cursor cursor)
+  public PooledTopNParams makeInitParams(ColumnSelectorPlus selectorPlus, Cursor cursor, CursorGranularizer granularizer)
   {
     final DimensionSelector dimSelector = (DimensionSelector) selectorPlus.getSelector();
     final int cardinality = dimSelector.getValueCardinality();
@@ -272,6 +273,7 @@ public class PooledTopNAlgorithm
       return PooledTopNParams.builder()
                              .withSelectorPlus(selectorPlus)
                              .withCursor(cursor)
+                             .withGranularizer(granularizer)
                              .withResultsBufHolder(resultsBufHolder)
                              .withResultsBuf(resultsBuf)
                              .withArrayProvider(arrayProvider)
@@ -387,8 +389,7 @@ public class PooledTopNAlgorithm
   private static long scanAndAggregateGeneric1Agg(
       PooledTopNParams params,
       int[] positions,
-      BufferAggregator aggregator,
-      Cursor cursor
+      BufferAggregator aggregator
   )
   {
     String runtimeShape = StringRuntimeShape.of(aggregator);
@@ -400,7 +401,8 @@ public class PooledTopNAlgorithm
         params.getDimSelector(),
         aggregator,
         params.getAggregatorSizes()[0],
-        cursor,
+        params.getCursor(),
+        params.getGranularizer(),
         positions,
         params.getResultsBuf()
     );
@@ -411,8 +413,7 @@ public class PooledTopNAlgorithm
   private static long scanAndAggregateGeneric2Agg(
       PooledTopNParams params,
       int[] positions,
-      BufferAggregator[] theAggregators,
-      Cursor cursor
+      BufferAggregator[] theAggregators
   )
   {
     String runtimeShape = StringRuntimeShape.of(theAggregators);
@@ -427,7 +428,8 @@ public class PooledTopNAlgorithm
         aggregatorSizes[0],
         theAggregators[1],
         aggregatorSizes[1],
-        cursor,
+        params.getCursor(),
+        params.getGranularizer(),
         positions,
         params.getResultsBuf()
     );
@@ -467,6 +469,7 @@ public class PooledTopNAlgorithm
     final int numBytesPerRecord = params.getNumBytesPerRecord();
     final int[] aggregatorSizes = params.getAggregatorSizes();
     final Cursor cursor = params.getCursor();
+    final CursorGranularizer granularizer = params.getGranularizer();
     final DimensionSelector dimSelector = params.getDimSelector();
 
     final int[] aggregatorOffsets = new int[aggregatorSizes.length];
@@ -666,8 +669,10 @@ public class PooledTopNAlgorithm
             currentPosition
         );
       }
-      cursor.advanceUninterruptibly();
       processedRows++;
+      if (!granularizer.advanceCursorWithinBucketUninterruptedly()) {
+        break;
+      }
     }
     return processedRows;
   }
@@ -799,6 +804,7 @@ public class PooledTopNAlgorithm
     public PooledTopNParams(
         ColumnSelectorPlus selectorPlus,
         Cursor cursor,
+        CursorGranularizer granularizer,
         ResourceHolder<ByteBuffer> resultsBufHolder,
         ByteBuffer resultsBuf,
         int[] aggregatorSizes,
@@ -807,7 +813,7 @@ public class PooledTopNAlgorithm
         TopNMetricSpecBuilder<int[]> arrayProvider
     )
     {
-      super(selectorPlus, cursor, numValuesPerPass);
+      super(selectorPlus, cursor, granularizer, numValuesPerPass);
 
       this.resultsBufHolder = resultsBufHolder;
       this.resultsBuf = resultsBuf;
@@ -850,6 +856,7 @@ public class PooledTopNAlgorithm
     {
       private ColumnSelectorPlus selectorPlus;
       private Cursor cursor;
+      private CursorGranularizer granularizer;
       private ResourceHolder<ByteBuffer> resultsBufHolder;
       private ByteBuffer resultsBuf;
       private int[] aggregatorSizes;
@@ -869,6 +876,12 @@ public class PooledTopNAlgorithm
         return this;
       }
 
+      public Builder withGranularizer(CursorGranularizer granularizer)
+      {
+        this.granularizer = granularizer;
+        return this;
+      }
+
       public Builder withResultsBufHolder(ResourceHolder<ByteBuffer> resultsBufHolder)
       {
         this.resultsBufHolder = resultsBufHolder;
@@ -910,6 +923,7 @@ public class PooledTopNAlgorithm
         return new PooledTopNParams(
             selectorPlus,
             cursor,
+            granularizer,
             resultsBufHolder,
             resultsBuf,
             aggregatorSizes,
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TimeExtractionTopNAlgorithm.java b/processing/src/main/java/org/apache/druid/query/topn/TimeExtractionTopNAlgorithm.java
index 3b60bb65ee1..7e5d43b96ad 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TimeExtractionTopNAlgorithm.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TimeExtractionTopNAlgorithm.java
@@ -20,6 +20,7 @@
 package org.apache.druid.query.topn;
 
 import org.apache.druid.query.ColumnSelectorPlus;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.DimensionHandlerUtils;
@@ -53,11 +54,12 @@ public class TimeExtractionTopNAlgorithm extends BaseTopNAlgorithm<int[], Map<Ob
 
   @Override
   @SuppressWarnings("unchecked")
-  public TopNParams makeInitParams(ColumnSelectorPlus selectorPlus, Cursor cursor)
+  public TopNParams makeInitParams(ColumnSelectorPlus selectorPlus, Cursor cursor, CursorGranularizer granularizer)
   {
     return new TopNParams(
         selectorPlus,
         cursor,
+        granularizer,
         Integer.MAX_VALUE
     );
   }
@@ -88,6 +90,7 @@ public class TimeExtractionTopNAlgorithm extends BaseTopNAlgorithm<int[], Map<Ob
   )
   {
     final Cursor cursor = params.getCursor();
+    final CursorGranularizer granularizer = params.getGranularizer();
     final DimensionSelector dimSelector = params.getDimSelector();
 
     long processedRows = 0;
@@ -102,9 +105,10 @@ public class TimeExtractionTopNAlgorithm extends BaseTopNAlgorithm<int[], Map<Ob
       for (Aggregator aggregator : theAggregators) {
         aggregator.aggregate();
       }
-
-      cursor.advance();
       processedRows++;
+      if (!granularizer.advanceCursorWithinBucket()) {
+        break;
+      }
     }
     return processedRows;
   }
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNAlgorithm.java b/processing/src/main/java/org/apache/druid/query/topn/TopNAlgorithm.java
index 31b4d9204e8..3f197aa8c8a 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNAlgorithm.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNAlgorithm.java
@@ -20,6 +20,7 @@
 package org.apache.druid.query.topn;
 
 import org.apache.druid.query.ColumnSelectorPlus;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.query.topn.types.TopNColumnAggregatesProcessor;
 import org.apache.druid.segment.Cursor;
@@ -34,7 +35,7 @@ public interface TopNAlgorithm<DimValSelector, Parameters extends TopNParams>
   int INIT_POSITION_VALUE = -1;
   int SKIP_POSITION_VALUE = -2;
 
-  TopNParams makeInitParams(ColumnSelectorPlus<TopNColumnAggregatesProcessor> selectorPlus, Cursor cursor);
+  TopNParams makeInitParams(ColumnSelectorPlus<TopNColumnAggregatesProcessor> selectorPlus, Cursor cursor, CursorGranularizer granularizer);
 
   void run(
       Parameters params,
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNMapFn.java b/processing/src/main/java/org/apache/druid/query/topn/TopNMapFn.java
index 96fb62f9012..dafff313afb 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNMapFn.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNMapFn.java
@@ -20,6 +20,7 @@
 package org.apache.druid.query.topn;
 
 import org.apache.druid.query.ColumnSelectorPlus;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.topn.types.TopNColumnAggregatesProcessor;
 import org.apache.druid.query.topn.types.TopNColumnAggregatesProcessorFactory;
@@ -44,7 +45,7 @@ public class TopNMapFn
 
   @SuppressWarnings("unchecked")
   @Nullable
-  public Result<TopNResultValue> apply(final Cursor cursor, final @Nullable TopNQueryMetrics queryMetrics)
+  public Result<TopNResultValue> apply(final Cursor cursor, final CursorGranularizer granularizer, final @Nullable TopNQueryMetrics queryMetrics)
   {
     final ColumnSelectorPlus<TopNColumnAggregatesProcessor<?>> selectorPlus =
         DimensionHandlerUtils.createColumnSelectorPlus(
@@ -59,7 +60,7 @@ public class TopNMapFn
 
     TopNParams params = null;
     try {
-      params = topNAlgorithm.makeInitParams(selectorPlus, cursor);
+      params = topNAlgorithm.makeInitParams(selectorPlus, cursor, granularizer);
       if (queryMetrics != null) {
         queryMetrics.columnValueSelector(selectorPlus.getSelector());
         queryMetrics.numValuesPerPass(params);
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNParams.java b/processing/src/main/java/org/apache/druid/query/topn/TopNParams.java
index cdc541f9924..3a932d00d28 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNParams.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNParams.java
@@ -20,6 +20,7 @@
 package org.apache.druid.query.topn;
 
 import org.apache.druid.query.ColumnSelectorPlus;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.topn.types.TopNColumnAggregatesProcessor;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.DimensionSelector;
@@ -30,6 +31,7 @@ public class TopNParams
 {
   public static final int CARDINALITY_UNKNOWN = -1;
   private final Cursor cursor;
+  private final CursorGranularizer granularizer;
   private final int cardinality;
   private final int numValuesPerPass;
   private final ColumnSelectorPlus<TopNColumnAggregatesProcessor> selectorPlus;
@@ -37,11 +39,13 @@ public class TopNParams
   protected TopNParams(
       ColumnSelectorPlus<TopNColumnAggregatesProcessor> selectorPlus,
       Cursor cursor,
+      CursorGranularizer granularizer,
       int numValuesPerPass
   )
   {
     this.selectorPlus = selectorPlus;
     this.cursor = cursor;
+    this.granularizer = granularizer;
     this.cardinality = selectorPlus.getColumnSelectorStrategy().getCardinality(selectorPlus.getSelector());
     this.numValuesPerPass = numValuesPerPass;
   }
@@ -63,6 +67,11 @@ public class TopNParams
     return cursor;
   }
 
+  public CursorGranularizer getGranularizer()
+  {
+    return granularizer;
+  }
+
   public int getCardinality()
   {
     return cardinality;
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java
index 21729022d61..2534c37f90f 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java
@@ -172,9 +172,8 @@ public class TopNQuery extends BaseQuery<Result<TopNResultValue>>
     return Queries.computeRequiredColumns(
         virtualColumns,
         dimFilter,
-        Collections.singletonList(dimensionSpec),
-        aggregatorSpecs,
-        Collections.emptyList()
+        Collections.singletonList(dimensionSpec.getDimension()),
+        aggregatorSpecs
     );
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryEngine.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryEngine.java
index 50b8a30d102..ac130d3fa86 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryEngine.java
@@ -19,16 +19,20 @@
 
 package org.apache.druid.query.topn;
 
-import com.google.common.base.Preconditions;
 import com.google.common.base.Predicates;
 import org.apache.druid.collections.NonBlockingPool;
-import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.collections.ResourceHolder;
+import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.query.CursorGranularizer;
+import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.extraction.ExtractionFn;
-import org.apache.druid.query.filter.Filter;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.SegmentMissingException;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.column.ColumnCapabilities;
@@ -36,11 +40,10 @@ import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.Types;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.filter.Filters;
-import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
 import java.nio.ByteBuffer;
-import java.util.List;
+import java.util.Collections;
 
 /**
  */
@@ -57,8 +60,8 @@ public class TopNQueryEngine
   /**
    * Do the thing - process a {@link StorageAdapter} into a {@link Sequence} of {@link TopNResultValue}, with one of the
    * fine {@link TopNAlgorithm} available chosen based on the type of column being aggregated. The algorithm provides a
-   * mapping function to process rows from the adapter {@link org.apache.druid.segment.Cursor} to apply
-   * {@link AggregatorFactory} and create or update {@link TopNResultValue}
+   * mapping function to process rows from the adapter {@link Cursor} to apply {@link AggregatorFactory} and create or
+   * update {@link TopNResultValue}
    */
   public Sequence<Result<TopNResultValue>> query(
       final TopNQuery query,
@@ -72,36 +75,36 @@ public class TopNQueryEngine
       );
     }
 
-    final List<Interval> queryIntervals = query.getQuerySegmentSpec().getIntervals();
-    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter()));
-    final Granularity granularity = query.getGranularity();
     final TopNMapFn mapFn = getMapFn(query, adapter, queryMetrics);
 
-    Preconditions.checkArgument(
-        queryIntervals.size() == 1,
-        "Can only handle a single interval, got[%s]",
-        queryIntervals
+    final CursorBuildSpec buildSpec = makeCursorBuildSpec(query, queryMetrics);
+    final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec);
+    final Cursor cursor = cursorHolder.asCursor();
+    if (cursor == null) {
+      return Sequences.withBaggage(Sequences.empty(), cursorHolder);
+    }
+    final CursorGranularizer granularizer = CursorGranularizer.create(
+        adapter,
+        cursor,
+        query.getGranularity(),
+        buildSpec.getInterval(),
+        false
     );
+    if (granularizer == null) {
+      return Sequences.withBaggage(Sequences.empty(), cursorHolder);
+    }
 
+    if (queryMetrics != null) {
+      queryMetrics.cursor(cursor);
+    }
     return Sequences.filter(
-        Sequences.map(
-            adapter.makeCursors(
-                filter,
-                queryIntervals.get(0),
-                query.getVirtualColumns(),
-                granularity,
-                query.isDescending(),
-                queryMetrics
-            ),
-            input -> {
-              if (queryMetrics != null) {
-                queryMetrics.cursor(input);
-              }
-              return mapFn.apply(input, queryMetrics);
-            }
-        ),
-        Predicates.notNull()
-    );
+        Sequences.simple(granularizer.getBucketIterable())
+                 .map(bucketInterval -> {
+                   granularizer.advanceToBucket(bucketInterval);
+                   return mapFn.apply(cursor, granularizer, queryMetrics);
+                 }),
+                 Predicates.notNull()
+    ).withBaggage(cursorHolder);
   }
 
   /**
@@ -132,13 +135,13 @@ public class TopNQueryEngine
 
 
     final TopNAlgorithm<?, ?> topNAlgorithm;
-    if (canUsePooledAlgorithm(selector, query, columnCapabilities)) {
+    if (canUsePooledAlgorithm(selector, query, columnCapabilities, bufferPool, cardinality, numBytesPerRecord)) {
       // pool based algorithm selection, if we can
       if (selector.isAggregateAllMetrics()) {
         // if sorted by dimension we should aggregate all metrics in a single pass, use the regular pooled algorithm for
         // this
         topNAlgorithm = new PooledTopNAlgorithm(adapter, query, bufferPool);
-      } else if (selector.isAggregateTopNMetricFirst() || query.context().getBoolean("doAggregateTopNMetricFirst", false)) {
+      } else if (shouldUseAggregateMetricFirstAlgorithm(query, selector)) {
         // for high cardinality dimensions with larger result sets we aggregate with only the ordering aggregation to
         // compute the first 'n' values, and then for the rest of the metrics but for only the 'n' values
         topNAlgorithm = new AggregateTopNMetricFirstAlgorithm(adapter, query, bufferPool);
@@ -173,7 +176,7 @@ public class TopNQueryEngine
    * algorithm) are optimized off-heap algorithms for aggregating dictionary encoded string columns. These algorithms
    * rely on dictionary ids being unique so to aggregate on the dictionary ids directly and defer
    * {@link org.apache.druid.segment.DimensionSelector#lookupName(int)} until as late as possible in query processing.
-   *
+   * <p>
    * When these conditions are not true, we have an on-heap fall-back algorithm, the {@link HeapBasedTopNAlgorithm}
    * (and {@link TimeExtractionTopNAlgorithm} for a specialized form for long columns) which aggregates on values of
    * selectors.
@@ -181,7 +184,10 @@ public class TopNQueryEngine
   private static boolean canUsePooledAlgorithm(
       final TopNAlgorithmSelector selector,
       final TopNQuery query,
-      final ColumnCapabilities capabilities
+      final ColumnCapabilities capabilities,
+      final NonBlockingPool<ByteBuffer> bufferPool,
+      final int cardinality,
+      final int numBytesPerRecord
   )
   {
     if (selector.isHasExtractionFn()) {
@@ -193,13 +199,54 @@ public class TopNQueryEngine
       // non-string output cannot use the pooled algorith, even if the underlying selector supports it
       return false;
     }
-    if (Types.is(capabilities, ValueType.STRING)) {
-      // string columns must use the on heap algorithm unless they have the following capabilites
-      return capabilities.isDictionaryEncoded().isTrue() && capabilities.areDictionaryValuesUnique().isTrue();
-    } else {
+    if (!Types.is(capabilities, ValueType.STRING)) {
       // non-strings are not eligible to use the pooled algorithm, and should use a heap algorithm
       return false;
     }
+
+    // string columns must use the on heap algorithm unless they have the following capabilites
+    if (!capabilities.isDictionaryEncoded().isTrue() || !capabilities.areDictionaryValuesUnique().isTrue()) {
+      return false;
+    }
+    if (Granularities.ALL.equals(query.getGranularity())) {
+      // all other requirements have been satisfied, ALL granularity can always use the pooled algorithms
+      return true;
+    }
+    // if not using ALL granularity, we can still potentially use the pooled algorithm if we are certain it doesn't
+    // need to make multiple passes (e.g. reset the cursor)
+    try (final ResourceHolder<ByteBuffer> resultsBufHolder = bufferPool.take()) {
+      final ByteBuffer resultsBuf = resultsBufHolder.get();
+
+      final int numBytesToWorkWith = resultsBuf.capacity();
+      final int numValuesPerPass = numBytesPerRecord > 0 ? numBytesToWorkWith / numBytesPerRecord : cardinality;
+
+      return numValuesPerPass <= cardinality;
+    }
+  }
+
+  private static boolean shouldUseAggregateMetricFirstAlgorithm(TopNQuery query, TopNAlgorithmSelector selector)
+  {
+    // must be using ALL granularity since it makes multiple passes and must reset the cursor
+    if (Granularities.ALL.equals(query.getGranularity())) {
+      return selector.isAggregateTopNMetricFirst() || query.context().getBoolean("doAggregateTopNMetricFirst", false);
+    }
+    return false;
+  }
+
+  public static CursorBuildSpec makeCursorBuildSpec(TopNQuery query, @Nullable QueryMetrics<?> queryMetrics)
+  {
+    return Granularities.decorateCursorBuildSpec(
+        query,
+        CursorBuildSpec.builder()
+                       .setInterval(query.getSingleInterval())
+                       .setFilter(Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter())))
+                       .setGroupingColumns(Collections.singletonList(query.getDimensionSpec().getDimension()))
+                       .setVirtualColumns(query.getVirtualColumns())
+                       .setAggregators(query.getAggregatorSpecs())
+                       .setQueryContext(query.context())
+                       .setQueryMetrics(queryMetrics)
+                       .build()
+    );
   }
 
   /**
diff --git a/processing/src/main/java/org/apache/druid/query/topn/types/NullableNumericTopNColumnAggregatesProcessor.java b/processing/src/main/java/org/apache/druid/query/topn/types/NullableNumericTopNColumnAggregatesProcessor.java
index 36253c8d069..4b56a833f8a 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/types/NullableNumericTopNColumnAggregatesProcessor.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/types/NullableNumericTopNColumnAggregatesProcessor.java
@@ -20,6 +20,7 @@
 package org.apache.druid.query.topn.types;
 
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.query.topn.BaseTopNAlgorithm;
 import org.apache.druid.query.topn.TopNParams;
@@ -88,6 +89,7 @@ public abstract class NullableNumericTopNColumnAggregatesProcessor<Selector exte
       TopNQuery query,
       Selector selector,
       Cursor cursor,
+      CursorGranularizer granularizer,
       Aggregator[][] rowSelector
   )
   {
@@ -106,8 +108,10 @@ public abstract class NullableNumericTopNColumnAggregatesProcessor<Selector exte
           aggregator.aggregate();
         }
       }
-      cursor.advance();
       processedRows++;
+      if (!granularizer.advanceCursorWithinBucket()) {
+        break;
+      }
     }
     return processedRows;
   }
diff --git a/processing/src/main/java/org/apache/druid/query/topn/types/StringTopNColumnAggregatesProcessor.java b/processing/src/main/java/org/apache/druid/query/topn/types/StringTopNColumnAggregatesProcessor.java
index c1afe548dbb..d6f716f2c2b 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/types/StringTopNColumnAggregatesProcessor.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/types/StringTopNColumnAggregatesProcessor.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.query.topn.types;
 
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.query.topn.BaseTopNAlgorithm;
 import org.apache.druid.query.topn.TopNParams;
@@ -112,6 +113,7 @@ public class StringTopNColumnAggregatesProcessor implements TopNColumnAggregates
       TopNQuery query,
       DimensionSelector selector,
       Cursor cursor,
+      CursorGranularizer granularizer,
       Aggregator[][] rowSelector
   )
   {
@@ -120,10 +122,11 @@ public class StringTopNColumnAggregatesProcessor implements TopNColumnAggregates
     // we must know cardinality to use array based aggregation. in cases where the same dictionary ids map to different
     // values (1:* or *:*), results can be entirely incorrect since an aggregator for a different value might be
     // chosen from the array based on the re-used dictionary id
-    if (notUnknown && hasDictionary) {
-      return scanAndAggregateWithCardinalityKnown(query, cursor, selector, rowSelector);
+    // finally, 'run' passes in selector as null for unknown case, so to be safe check for null
+    if (notUnknown && hasDictionary && rowSelector != null) {
+      return scanAndAggregateWithCardinalityKnown(query, cursor, granularizer, selector, rowSelector);
     } else {
-      return scanAndAggregateWithCardinalityUnknown(query, cursor, selector);
+      return scanAndAggregateWithCardinalityUnknown(query, cursor, granularizer, selector);
     }
   }
 
@@ -141,6 +144,7 @@ public class StringTopNColumnAggregatesProcessor implements TopNColumnAggregates
   private long scanAndAggregateWithCardinalityKnown(
       TopNQuery query,
       Cursor cursor,
+      CursorGranularizer granularizer,
       DimensionSelector selector,
       Aggregator[][] rowSelector
   )
@@ -164,8 +168,10 @@ public class StringTopNColumnAggregatesProcessor implements TopNColumnAggregates
           aggregator.aggregate();
         }
       }
-      cursor.advance();
       processedRows++;
+      if (!granularizer.advanceCursorWithinBucket()) {
+        break;
+      }
     }
     return processedRows;
   }
@@ -181,6 +187,7 @@ public class StringTopNColumnAggregatesProcessor implements TopNColumnAggregates
   private long scanAndAggregateWithCardinalityUnknown(
       TopNQuery query,
       Cursor cursor,
+      CursorGranularizer granularizer,
       DimensionSelector selector
   )
   {
@@ -198,8 +205,10 @@ public class StringTopNColumnAggregatesProcessor implements TopNColumnAggregates
           aggregator.aggregate();
         }
       }
-      cursor.advance();
       processedRows++;
+      if (!granularizer.advanceCursorWithinBucket()) {
+        break;
+      }
     }
     return processedRows;
   }
diff --git a/processing/src/main/java/org/apache/druid/query/topn/types/TopNColumnAggregatesProcessor.java b/processing/src/main/java/org/apache/druid/query/topn/types/TopNColumnAggregatesProcessor.java
index d7c2fcd5b2b..5fd141e6428 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/types/TopNColumnAggregatesProcessor.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/types/TopNColumnAggregatesProcessor.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.query.topn.types;
 
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.query.dimension.ColumnSelectorStrategy;
 import org.apache.druid.query.topn.HeapBasedTopNAlgorithm;
@@ -96,6 +97,7 @@ public interface TopNColumnAggregatesProcessor<ValueSelectorType> extends Column
       TopNQuery query,
       ValueSelectorType selector,
       Cursor cursor,
+      CursorGranularizer granularizer,
       Aggregator[][] rowSelector
   );
 
diff --git a/processing/src/main/java/org/apache/druid/query/vector/VectorCursorGranularizer.java b/processing/src/main/java/org/apache/druid/query/vector/VectorCursorGranularizer.java
index 163befcf281..d899e1f0888 100644
--- a/processing/src/main/java/org/apache/druid/query/vector/VectorCursorGranularizer.java
+++ b/processing/src/main/java/org/apache/druid/query/vector/VectorCursorGranularizer.java
@@ -31,43 +31,15 @@ import org.joda.time.Interval;
 import javax.annotation.Nullable;
 
 /**
- * Class that helps vectorized query engines handle "granularity" parameters. Nonvectorized engines have it handled
- * for them by the StorageAdapter. Vectorized engines don't, because they can get efficiency gains by pushing
- * granularity handling into the engine layer.
+ * Class that helps vectorized query engines handle "granularity" parameters. Given a set of intervals, this class
+ * provides mechansims to advance a cursor to the start of an interval ({@link #setCurrentOffsets(Interval)}),
+ * advance a cursor within a bucket interval ({@link #advanceCursorWithinBucket()}), and check the offsets of the
+ * current vector that are within the bucket ({@link #getStartOffset()}, {@link #getEndOffset()}).
+ *
+ * @see org.apache.druid.query.CursorGranularizer for non-vectorized query engines.
  */
 public class VectorCursorGranularizer
 {
-  // And a cursor that has been made from it.
-  private final VectorCursor cursor;
-
-  // Iterable that iterates over time buckets.
-  private final Iterable<Interval> bucketIterable;
-
-  // Vector selector for the "__time" column.
-  @Nullable
-  private final VectorValueSelector timeSelector;
-
-  // Current time vector.
-  @Nullable
-  private long[] timestamps = null;
-
-  // Offset into the vector that we should start reading from.
-  private int startOffset = 0;
-
-  // Offset into the vector that is one past the last one we should read.
-  private int endOffset = 0;
-
-  private VectorCursorGranularizer(
-      VectorCursor cursor,
-      Iterable<Interval> bucketIterable,
-      @Nullable VectorValueSelector timeSelector
-  )
-  {
-    this.cursor = cursor;
-    this.bucketIterable = bucketIterable;
-    this.timeSelector = timeSelector;
-  }
-
   @Nullable
   public static VectorCursorGranularizer create(
       final StorageAdapter storageAdapter,
@@ -102,6 +74,36 @@ public class VectorCursorGranularizer
     return new VectorCursorGranularizer(cursor, bucketIterable, timeSelector);
   }
 
+  private final VectorCursor cursor;
+
+  // Iterable that iterates over time buckets.
+  private final Iterable<Interval> bucketIterable;
+
+  // Vector selector for the "__time" column.
+  @Nullable
+  private final VectorValueSelector timeSelector;
+
+  // Current time vector.
+  @Nullable
+  private long[] timestamps = null;
+
+  // Offset into the vector that we should start reading from.
+  private int startOffset = 0;
+
+  // Offset into the vector that is one past the last one we should read.
+  private int endOffset = 0;
+
+  private VectorCursorGranularizer(
+      VectorCursor cursor,
+      Iterable<Interval> bucketIterable,
+      @Nullable VectorValueSelector timeSelector
+  )
+  {
+    this.cursor = cursor;
+    this.bucketIterable = bucketIterable;
+    this.timeSelector = timeSelector;
+  }
+
   public void setCurrentOffsets(final Interval bucketInterval)
   {
     final long timeStart = bucketInterval.getStartMillis();
diff --git a/processing/src/main/java/org/apache/druid/segment/ArrayListSegment.java b/processing/src/main/java/org/apache/druid/segment/ArrayListSegment.java
index e7423b5b89d..4fa8d3b45b3 100644
--- a/processing/src/main/java/org/apache/druid/segment/ArrayListSegment.java
+++ b/processing/src/main/java/org/apache/druid/segment/ArrayListSegment.java
@@ -46,8 +46,8 @@ public class ArrayListSegment<RowType> implements Segment
    * Create a list-based segment.
    * <p>
    * The provided List must be in time-order according to the provided {@link RowAdapter#timestampFunction()}.
-   * The cursor returned by {@link RowBasedStorageAdapter#makeCursors} makes no attempt to verify this, and callers
-   * will expect it.
+   * The cursor returned by {@link RowBasedStorageAdapter#makeCursorHolder(CursorBuildSpec)} makes no attempt to verify
+   * this, and callers will expect it.
    * <p>
    * The provided "rowSignature" will be used for reporting available columns and their capabilities to users of
    * {@link #asStorageAdapter()}. Note that the {@link ColumnSelectorFactory} implementation returned by this segment's
diff --git a/processing/src/main/java/org/apache/druid/segment/BitmapOffset.java b/processing/src/main/java/org/apache/druid/segment/BitmapOffset.java
index ab552f0d5a3..94324fe2206 100644
--- a/processing/src/main/java/org/apache/druid/segment/BitmapOffset.java
+++ b/processing/src/main/java/org/apache/druid/segment/BitmapOffset.java
@@ -23,13 +23,13 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap;
 import org.apache.druid.collections.bitmap.MutableBitmap;
 import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
 import org.apache.druid.collections.bitmap.WrappedImmutableRoaringBitmap;
-import org.apache.druid.extendedset.intset.EmptyIntIterator;
 import org.apache.druid.java.util.common.RE;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.data.Offset;
 import org.apache.druid.segment.data.ReadableOffset;
 import org.roaringbitmap.IntIterator;
 
+import javax.annotation.Nullable;
 import java.util.Arrays;
 import java.util.HashSet;
 
@@ -126,6 +126,9 @@ public class BitmapOffset extends Offset
   private final IntIterator iteratorForReset;
   private final int valueForReset;
   private int value;
+  @Nullable
+  private IntIterator iteratorForMark;
+  private int valueForMark;
 
   public static IntIterator getReverseBitmapOffsetIterator(ImmutableBitmap bitmapIndex)
   {
@@ -153,7 +156,7 @@ public class BitmapOffset extends Offset
     increment();
     // It's important to set iteratorForReset and valueForReset after calling increment(), because only after that the
     // iterator and the value are in proper initial state.
-    this.iteratorForReset = safeClone(iterator);
+    this.iteratorForReset = iterator.clone();
     this.valueForReset = value;
   }
 
@@ -169,13 +172,21 @@ public class BitmapOffset extends Offset
   /**
    * Constructor for {@link #clone()}.
    */
-  private BitmapOffset(String fullness, IntIterator iterator, int value)
+  private BitmapOffset(
+      String fullness,
+      IntIterator iterator,
+      int value,
+      @Nullable IntIterator iteratorForMark,
+      int markValue
+  )
   {
     this.fullness = fullness;
     this.iterator = iterator;
-    this.iteratorForReset = safeClone(iterator);
+    this.iteratorForReset = iterator.clone();
     this.valueForReset = value;
     this.value = value;
+    this.iteratorForMark = iteratorForMark;
+    this.valueForMark = markValue;
   }
 
   @Override
@@ -197,8 +208,10 @@ public class BitmapOffset extends Offset
   @Override
   public void reset()
   {
-    iterator = safeClone(iteratorForReset);
+    iterator = iteratorForReset.clone();
     value = valueForReset;
+    valueForMark = valueForReset;
+    iteratorForMark = null;
   }
 
   @Override
@@ -211,7 +224,13 @@ public class BitmapOffset extends Offset
   @Override
   public Offset clone()
   {
-    return new BitmapOffset(fullness, safeClone(iterator), value);
+    return new BitmapOffset(
+        fullness,
+        iterator.clone(),
+        value,
+        iteratorForMark != null ? iteratorForMark.clone() : null,
+        valueForMark
+    );
   }
 
   @Override
@@ -226,11 +245,4 @@ public class BitmapOffset extends Offset
     inspector.visit("iterator", iterator);
     inspector.visit("fullness", fullness);
   }
-
-  private static IntIterator safeClone(IntIterator iterator)
-  {
-    // Calling clone() on empty iterators from RoaringBitmap library sometimes fails with NPE,
-    // see https://github.com/apache/druid/issues/4709, https://github.com/RoaringBitmap/RoaringBitmap/issues/177
-    return iterator.hasNext() ? iterator.clone() : EmptyIntIterator.instance();
-  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/Cursor.java b/processing/src/main/java/org/apache/druid/segment/Cursor.java
index 925c7387e53..a1fc14c1054 100644
--- a/processing/src/main/java/org/apache/druid/segment/Cursor.java
+++ b/processing/src/main/java/org/apache/druid/segment/Cursor.java
@@ -19,14 +19,33 @@
 
 package org.apache.druid.segment;
 
-import org.joda.time.DateTime;
+import org.apache.druid.query.QueryInterruptedException;
+import org.apache.druid.segment.incremental.IncrementalIndexCursorHolder;
 
 /**
- * Cursor is an interface for iteration over a range of data points, used during query execution. {@link
- * QueryableIndexCursorSequenceBuilder.QueryableIndexCursor} is an implementation for historical segments, and {@link
- * org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter.IncrementalIndexCursor} is an implementation for {@link
- * org.apache.druid.segment.incremental.IncrementalIndex}.
+ * Cursor is an interface for iteration over a range of data points, used during query execution. Cursors are available
+ * from {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} via {@link CursorHolder#asCursor()}.
+ * <p>
+ * A typical usage pattern might look something like this:
+ * <pre>
+ *   try (CursorHolder cursorHolder = adapter.makeCursorHolder(...)) {
+ *     Cursor cursor = cursorHolder.asCursor();
+ *     ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
+ *     ColumnValueSelector timeSelector = factory.makeColumnValueSelector("__time");
+ *     // ...
+ *     while (!cursor.isDone()) {}
+ *       long time = timeSelector.getLong();
+ *       // do stuff with column values
+ *       // ...
+ *       cursor.advance();
+ *     }
+ *   }
+ * </pre>
  *
+ * {@link QueryableIndexCursorHolder.QueryableIndexCursor} is an implementation for historical segments, and
+ * {@link IncrementalIndexCursorHolder.IncrementalIndexCursor} is an implementation for
+ * {@link org.apache.druid.segment.incremental.IncrementalIndex}.
+ * <p>
  * Cursor is conceptually similar to {@link TimeAndDimsPointer}, but the latter is used for historical segment creation
  * rather than query execution (as Cursor). If those abstractions could be collapsed (and if it is worthwhile) is yet to
  * be determined.
@@ -35,11 +54,42 @@ import org.joda.time.DateTime;
  */
 public interface Cursor
 {
+  /**
+   * Get a {@link ColumnSelectorFactory} whose selectors will be backed by the row values at the current position of
+   * the cursor
+   */
   ColumnSelectorFactory getColumnSelectorFactory();
-  DateTime getTime();
+
+  /**
+   * Advance the cursor to the next position, checking if thread has been interrupted after advancing and possibly
+   * throwing {@link QueryInterruptedException} if so. Callers should check {@link #isDone()} or
+   * {@link #isDoneOrInterrupted()} before getting the next value from a selector.
+   */
   void advance();
+
+  /**
+   * Advance to the cursor to the next position. Callers should check {@link #isDone()} or
+   * {@link #isDoneOrInterrupted()} before getting the next value from a selector.
+   */
   void advanceUninterruptibly();
+
+  /**
+   * Check if the current cursor position is valid, returning false if there are values to read from selectors created
+   * by {@link #getColumnSelectorFactory()}. If true, any such selectors will no longer produce values.
+   */
   boolean isDone();
+
+  /**
+   * Check if the current cursor position is valid, or if the thread has been interrupted.
+   *
+   * @see #isDone()
+   */
   boolean isDoneOrInterrupted();
+
+  /**
+   * Reset to start of cursor. Most cursor implementations are backed by immutable data, but there is generically no
+   * guarantee that advancing through a cursor again will read exactly the same data or even number of rows, since the
+   * underlying data might be mutable in some cases.
+   */
   void reset();
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
new file mode 100644
index 00000000000..f6cbbdeffe8
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.query.QueryMetrics;
+import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.query.filter.Filter;
+import org.joda.time.Interval;
+
+import javax.annotation.Nullable;
+import java.util.List;
+
+public class CursorBuildSpec
+{
+  public static final CursorBuildSpec FULL_SCAN = builder().build();
+
+  public static CursorBuildSpecBuilder builder()
+  {
+    return new CursorBuildSpecBuilder();
+  }
+
+  public static CursorBuildSpecBuilder builder(CursorBuildSpec spec)
+  {
+    return new CursorBuildSpecBuilder(spec);
+  }
+
+  @Nullable
+  private final Filter filter;
+  private final Interval interval;
+  @Nullable
+  private final List<String> groupingColumns;
+  private final VirtualColumns virtualColumns;
+  @Nullable
+  private final List<AggregatorFactory> aggregators;
+  @Nullable
+  private final List<OrderBy> orderByColumns;
+
+  private final QueryContext queryContext;
+
+  @Nullable
+  private final QueryMetrics<?> queryMetrics;
+
+  public CursorBuildSpec(
+      @Nullable Filter filter,
+      Interval interval,
+      @Nullable List<String> groupingColumns,
+      VirtualColumns virtualColumns,
+      @Nullable List<AggregatorFactory> aggregators,
+      @Nullable List<OrderBy> preferredOrdering,
+      QueryContext queryContext,
+      @Nullable QueryMetrics<?> queryMetrics
+  )
+  {
+    this.filter = filter;
+    this.interval = interval;
+    this.groupingColumns = groupingColumns;
+    this.virtualColumns = virtualColumns;
+    this.aggregators = aggregators;
+    this.orderByColumns = preferredOrdering;
+    this.queryContext = queryContext;
+    this.queryMetrics = queryMetrics;
+  }
+
+  /**
+   * {@link Filter} to supply to the {@link CursorHolder}. Only rows which match will be available through the
+   * selectors created from the {@link Cursor} or {@link org.apache.druid.segment.vector.VectorCursor}
+   */
+  @Nullable
+  public Filter getFilter()
+  {
+    return filter;
+  }
+
+  /**
+   * {@link Interval} filter to supply to the {@link CursorHolder}. Only rows whose timestamps fall within this range
+   * will be available through the selectors created from the {@link Cursor} or
+   * {@link org.apache.druid.segment.vector.VectorCursor}
+   */
+  public Interval getInterval()
+  {
+    return interval;
+  }
+
+  /**
+   * Any columns which will be used for grouping by a query engine for the {@link CursorHolder}, useful for
+   * specializing the {@link Cursor} or {@link org.apache.druid.segment.vector.VectorCursor} if any pre-aggregated
+   * data is available.
+   */
+  @Nullable
+  public List<String> getGroupingColumns()
+  {
+    return groupingColumns;
+  }
+
+  /**
+   * Any {@link VirtualColumns} which are used by a query engine to assist in
+   * determining if {@link CursorHolder#canVectorize()}
+   */
+  public VirtualColumns getVirtualColumns()
+  {
+    return virtualColumns;
+  }
+
+  /**
+   * Any {@link AggregatorFactory} which will be used by a query engine for the {@link CursorHolder}, useful
+   * to assist in determining if {@link CursorHolder#canVectorize()}, as well as specializing the {@link Cursor} or
+   * {@link org.apache.druid.segment.vector.VectorCursor} if any pre-aggregated data is available.
+   */
+  @Nullable
+  public List<AggregatorFactory> getAggregators()
+  {
+    return aggregators;
+  }
+
+  /**
+   * List of all {@link OrderBy} columns which a query engine will use to sort its results to supply to the
+   * {@link CursorHolder}, which can allow optimization of the provided {@link Cursor} or
+   * {@link org.apache.druid.segment.vector.VectorCursor} if data matching the preferred ordering is available.
+   * <p>
+   * If not specified, the cursor will advance in the native order of the underlying data.
+   */
+  @Nullable
+  public List<OrderBy> getPreferredOrdering()
+  {
+    return orderByColumns;
+  }
+
+  /**
+   * {@link QueryContext} for the {@link CursorHolder} to provide a mechanism to push various data into
+   * {@link Cursor} and {@link org.apache.druid.segment.vector.VectorCursor} such as
+   * {@link org.apache.druid.query.QueryContexts#VECTORIZE_KEY} and
+   * {@link org.apache.druid.query.QueryContexts#VECTOR_SIZE_KEY}
+   */
+  public QueryContext getQueryContext()
+  {
+    return queryContext;
+  }
+
+  /**
+   * {@link QueryMetrics} to use for measuring things involved with {@link Cursor} and
+   * {@link org.apache.druid.segment.vector.VectorCursor} creation.
+   */
+  @Nullable
+  public QueryMetrics<?> getQueryMetrics()
+  {
+    return queryMetrics;
+  }
+
+  public static class CursorBuildSpecBuilder
+  {
+    @Nullable
+    private Filter filter;
+    private Interval interval = Intervals.ETERNITY;
+
+    @Nullable
+    private List<String> groupingColumns;
+    private VirtualColumns virtualColumns = VirtualColumns.EMPTY;
+    @Nullable
+    private List<AggregatorFactory> aggregators;
+    @Nullable
+    private List<OrderBy> preferredOrdering;
+
+    private QueryContext queryContext = QueryContext.empty();
+    @Nullable
+    private QueryMetrics<?> queryMetrics;
+
+    private CursorBuildSpecBuilder()
+    {
+      // initialize with defaults
+    }
+
+    private CursorBuildSpecBuilder(CursorBuildSpec buildSpec)
+    {
+      this.filter = buildSpec.filter;
+      this.interval = buildSpec.interval;
+      this.groupingColumns = buildSpec.groupingColumns;
+      this.virtualColumns = buildSpec.virtualColumns;
+      this.aggregators = buildSpec.aggregators;
+      this.preferredOrdering = buildSpec.orderByColumns;
+      this.queryContext = buildSpec.queryContext;
+      this.queryMetrics = buildSpec.queryMetrics;
+    }
+
+    /**
+     * @see CursorBuildSpec#getFilter()
+     */
+    public CursorBuildSpecBuilder setFilter(@Nullable Filter filter)
+    {
+      this.filter = filter;
+      return this;
+    }
+
+    /**
+     * @see CursorBuildSpec#getInterval()
+     */
+    public CursorBuildSpecBuilder setInterval(Interval interval)
+    {
+      this.interval = interval;
+      return this;
+    }
+
+    /**
+     * @see CursorBuildSpec#getGroupingColumns()
+     */
+    public CursorBuildSpecBuilder setGroupingColumns(
+        @Nullable List<String> groupingColumns
+    )
+    {
+      this.groupingColumns = groupingColumns;
+      return this;
+    }
+
+    /**
+     * @see CursorBuildSpec#getVirtualColumns()
+     */
+    public CursorBuildSpecBuilder setVirtualColumns(VirtualColumns virtualColumns)
+    {
+      this.virtualColumns = virtualColumns;
+      return this;
+    }
+
+    /**
+     * @see CursorBuildSpec#getAggregators()
+     */
+    public CursorBuildSpecBuilder setAggregators(@Nullable List<AggregatorFactory> aggregators)
+    {
+      this.aggregators = aggregators;
+      return this;
+    }
+
+    /**
+     * @see CursorBuildSpec#getPreferredOrdering()
+     */
+    public CursorBuildSpecBuilder setPreferredOrdering(@Nullable List<OrderBy> orderBy)
+    {
+      this.preferredOrdering = orderBy;
+      return this;
+    }
+
+    /**
+     * @see CursorBuildSpec#getQueryContext()
+     */
+    public CursorBuildSpecBuilder setQueryContext(QueryContext queryContext)
+    {
+      this.queryContext = queryContext;
+      return this;
+    }
+
+    /**
+     * @see CursorBuildSpec#getQueryMetrics()
+     */
+    public CursorBuildSpecBuilder setQueryMetrics(@Nullable QueryMetrics<?> queryMetrics)
+    {
+      this.queryMetrics = queryMetrics;
+      return this;
+    }
+
+    public CursorBuildSpec build()
+    {
+      return new CursorBuildSpec(
+          filter,
+          interval,
+          groupingColumns,
+          virtualColumns,
+          aggregators,
+          preferredOrdering,
+          queryContext,
+          queryMetrics
+      );
+    }
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/CursorFactory.java b/processing/src/main/java/org/apache/druid/segment/CursorFactory.java
index 809e7b85b06..551430a00e3 100644
--- a/processing/src/main/java/org/apache/druid/segment/CursorFactory.java
+++ b/processing/src/main/java/org/apache/druid/segment/CursorFactory.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.segment;
 
+import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.QueryMetrics;
@@ -32,7 +33,12 @@ import javax.annotation.Nullable;
  * Interface extended by {@link StorageAdapter}, which gives them the power to create cursors.
  *
  * @see StorageAdapter
+ *
+ * @deprecated This interface is deprecated and no longer implemented by any built-in {@link StorageAdapter}. Callers
+ * should use {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} instead. Implementors should implement
+ * {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} instead.
  */
+@Deprecated
 public interface CursorFactory
 {
   /**
@@ -40,27 +46,46 @@ public interface CursorFactory
    *
    * Query engines should use this before running in vectorized mode, and be prepared to fall back to non-vectorized
    * mode if this method returns false.
+   *
+   * @deprecated Callers should use {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} and call
+   * {@link CursorHolder#canVectorize()}.
+   * Implementors should implement {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} instead.  This method is
+   * no longer implemented by any built-in factories.
    */
+  @Deprecated
   default boolean canVectorize(
       @Nullable Filter filter,
       VirtualColumns virtualColumns,
       boolean descending
   )
   {
-    return false;
+    throw DruidException.defensive(
+        "CursorFactory.canVectorize is no longer supported, use CursorHolderFactory.makeCursorHolder instead"
+    );
   }
 
   /**
    * Creates a sequence of Cursors, one for each time-granular bucket (based on the provided Granularity).
+   *
+   * @deprecated Callers should use {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} and call
+   * {@link CursorHolder#asCursor()}.
+   * Implementors should implement {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} instead.
+   * This method is no longer implemented by any built-in factories.
    */
-  Sequence<Cursor> makeCursors(
+  @Deprecated
+  default Sequence<Cursor> makeCursors(
       @Nullable Filter filter,
       Interval interval,
       VirtualColumns virtualColumns,
       Granularity gran,
       boolean descending,
       @Nullable QueryMetrics<?> queryMetrics
-  );
+  )
+  {
+    throw DruidException.defensive(
+        "CursorFactory.makeCursors is no longer supported, use CursorHolderFactory.makeCursorHolder instead"
+    );
+  }
 
   /**
    * Creates a VectorCursor. Unlike the Cursor returned by "makeCursor", there is just one of these. Hence, this method
@@ -69,7 +94,13 @@ public interface CursorFactory
    *
    * Returns null if there is no data to walk over (for example, if the "interval" does not overlap the data interval
    * of this segment).
+   *
+   * @deprecated Callers should use {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} and call
+   * {@link CursorHolder#asVectorCursor()}. Implementors should implement
+   * {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} instead. This method is no longer implemented by any
+   * built-in factories.
    */
+  @Deprecated
   @Nullable
   default VectorCursor makeVectorCursor(
       @Nullable Filter filter,
@@ -80,6 +111,8 @@ public interface CursorFactory
       @Nullable QueryMetrics<?> queryMetrics
   )
   {
-    throw new UnsupportedOperationException("Cannot vectorize. Check 'canVectorize' before calling 'makeVectorCursor'.");
+    throw DruidException.defensive(
+        "CursorFactory.makeVectorCursor is no longer supported, use CursorHolderFactory.makeCursorHolder instead"
+    );
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/CursorHolder.java b/processing/src/main/java/org/apache/druid/segment/CursorHolder.java
new file mode 100644
index 00000000000..d0384e265b3
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/CursorHolder.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import org.apache.druid.java.util.common.UOE;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.segment.vector.VectorCursor;
+
+import javax.annotation.Nullable;
+import java.io.Closeable;
+import java.util.Collections;
+import java.util.List;
+
+public interface CursorHolder extends Closeable
+{
+  /**
+   * Create a {@link Cursor} for use with non-vectorized query engines.
+   */
+  @Nullable
+  Cursor asCursor();
+
+  /**
+   * Create a {@link VectorCursor} for use with vectorized query engines.
+   */
+  @Nullable
+  default VectorCursor asVectorCursor()
+  {
+    throw new UOE("Cannot vectorize. Check 'canVectorize' before calling 'makeVectorCursor' on %s.", this.getClass().getName());
+  }
+
+  /**
+   * Returns true if this {@link CursorHolder} supports calling {@link #asVectorCursor()}.
+   */
+  default boolean canVectorize()
+  {
+    return false;
+  }
+
+  /**
+   * Returns cursor ordering, which may or may not match {@link CursorBuildSpec#getPreferredOrdering()}. If returns
+   * null then the cursor has no defined ordering
+   */
+  default List<OrderBy> getOrdering()
+  {
+    return Collections.emptyList();
+  }
+
+  /**
+   * Release any resources acquired by cursors.
+   */
+  @Override
+  default void close()
+  {
+    // nothing to close
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/CursorHolderFactory.java b/processing/src/main/java/org/apache/druid/segment/CursorHolderFactory.java
new file mode 100644
index 00000000000..e191f033f59
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/CursorHolderFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+public interface CursorHolderFactory
+{
+  CursorHolder makeCursorHolder(CursorBuildSpec spec);
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/Cursors.java b/processing/src/main/java/org/apache/druid/segment/Cursors.java
new file mode 100644
index 00000000000..814681ab8b4
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/Cursors.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import org.apache.druid.error.DruidException;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.segment.column.ColumnHolder;
+
+import java.util.Collections;
+import java.util.List;
+
+public class Cursors
+{
+  private static final List<OrderBy> TIME_ASCENDING_ORDER = Collections.singletonList(
+      OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME)
+  );
+
+  private static final List<OrderBy> TIME_DESCENDING_ORDER = Collections.singletonList(
+      OrderBy.descending(ColumnHolder.TIME_COLUMN_NAME)
+  );
+
+  /**
+   * Check if the first {@link OrderBy} column of {@link CursorBuildSpec#getPreferredOrdering()} is
+   * {@link Order#DESCENDING}, which allow {@link Cursor} on time ordered data to advance in descending order if
+   * possible.
+   */
+  public static boolean preferDescendingTimeOrdering(CursorBuildSpec buildSpec)
+  {
+    final List<OrderBy> preferredOrdering = buildSpec.getPreferredOrdering();
+    if (preferredOrdering != null && !preferredOrdering.isEmpty()) {
+      final OrderBy orderBy = preferredOrdering.get(0);
+      return ColumnHolder.TIME_COLUMN_NAME.equals(orderBy.getColumnName()) && Order.DESCENDING == orderBy.getOrder();
+    }
+    return false;
+  }
+
+  /**
+   * Check if the first {@link OrderBy} of {@link CursorHolder#getOrdering()} is {@link ColumnHolder#TIME_COLUMN_NAME}
+   * in the specified {@link Order}
+   */
+  public static boolean isTimeOrdered(CursorHolder holder, Order order)
+  {
+    final List<OrderBy> ordering = holder.getOrdering();
+    if (ordering.isEmpty()) {
+      return false;
+    }
+    final OrderBy orderBy = ordering.get(0);
+    return ColumnHolder.TIME_COLUMN_NAME.equals(orderBy.getColumnName()) && orderBy.getOrder() == order;
+  }
+
+  /**
+   * Require the first {@link OrderBy} of {@link CursorHolder#getOrdering()} is {@link ColumnHolder#TIME_COLUMN_NAME}.
+   * Throws {@link DruidException} if the order does
+   */
+  public static void requireTimeOrdering(CursorHolder holder, Order order)
+  {
+    if (!isTimeOrdered(holder, order)) {
+      final String failureReason = StringUtils.format(
+          "Cursor must be ordered by [%s] with direction [%s] but was [%s] instead.",
+          ColumnHolder.TIME_COLUMN_NAME,
+          order,
+          holder.getOrdering()
+      );
+      holder.close();
+      throw DruidException.defensive(failureReason);
+    }
+  }
+
+  /**
+   * Get a {@link CursorHolder} {@link OrderBy} list that contains only a {@link ColumnHolder#TIME_COLUMN_NAME} as
+   * {@link Order#ASCENDING}, classic Druid segment order.
+   */
+  public static List<OrderBy> ascendingTimeOrder()
+  {
+    return TIME_ASCENDING_ORDER;
+  }
+
+  /**
+   * Get a {@link CursorHolder} {@link OrderBy} list that contains only a {@link ColumnHolder#TIME_COLUMN_NAME} as
+   * {@link Order#DESCENDING}, classic Druid segment order in reverse.
+   */
+  public static List<OrderBy> descendingTimeOrder()
+  {
+    return TIME_DESCENDING_ORDER;
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
index 1168e34b28a..b4b0ddfbb60 100644
--- a/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
@@ -20,9 +20,6 @@
 package org.apache.druid.segment;
 
 import com.google.common.collect.ImmutableList;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.column.ColumnCapabilities;
@@ -35,36 +32,36 @@ import javax.annotation.Nullable;
 
 public class FilteredStorageAdapter implements StorageAdapter
 {
+  @Nullable
   private final DimFilter filterOnDataSource;
   private final StorageAdapter baseStorageAdapter;
 
-  public FilteredStorageAdapter(final StorageAdapter adapter, final DimFilter filter)
+  public FilteredStorageAdapter(final StorageAdapter adapter, @Nullable final DimFilter filter)
   {
     this.baseStorageAdapter = adapter;
     this.filterOnDataSource = filter;
   }
 
   @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      Granularity gran,
-      boolean descending,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    final Filter andFilter;
-    if (filter == null) {
+    final CursorBuildSpec.CursorBuildSpecBuilder buildSpecBuilder = CursorBuildSpec.builder(spec);
+    final Filter newFilter;
+    if (spec.getFilter() == null) {
       if (filterOnDataSource != null) {
-        andFilter = filterOnDataSource.toFilter();
+        newFilter = filterOnDataSource.toFilter();
       } else {
-        andFilter = null;
+        newFilter = null;
       }
     } else {
-      andFilter = new AndFilter(ImmutableList.of(filter, filterOnDataSource.toFilter()));
+      if (filterOnDataSource != null) {
+        newFilter = new AndFilter(ImmutableList.of(spec.getFilter(), filterOnDataSource.toFilter()));
+      } else {
+        newFilter = spec.getFilter();
+      }
     }
-    return baseStorageAdapter.makeCursors(andFilter, interval, virtualColumns, gran, descending, queryMetrics);
+    buildSpecBuilder.setFilter(newFilter);
+    return baseStorageAdapter.makeCursorHolder(buildSpecBuilder.build());
   }
 
   @Override
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
similarity index 52%
rename from processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java
rename to processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
index 86069068eaf..0f0646839ac 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
@@ -20,21 +20,20 @@
 package org.apache.druid.segment;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
 import org.apache.druid.collections.bitmap.BitmapFactory;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.BitmapResultFactory;
 import org.apache.druid.query.DefaultBitmapResultFactory;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
+import org.apache.druid.query.QueryContext;
 import org.apache.druid.query.QueryMetrics;
+import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.FilterBundle;
 import org.apache.druid.query.filter.ValueMatcher;
@@ -53,65 +52,102 @@ import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
 import org.apache.druid.segment.vector.VectorCursor;
 import org.apache.druid.segment.vector.VectorOffset;
 import org.apache.druid.utils.CloseableUtils;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
+import java.io.Closeable;
 import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-public class QueryableIndexCursorSequenceBuilder
+public class QueryableIndexCursorHolder implements CursorHolder
 {
-  private static final Logger log = new Logger(QueryableIndexCursorSequenceBuilder.class);
+  private static final Logger log = new Logger(QueryableIndexCursorHolder.class);
   private final QueryableIndex index;
   private final Interval interval;
   private final VirtualColumns virtualColumns;
+
+  @Nullable
+  private final List<AggregatorFactory> aggregatorFactories;
   @Nullable
   private final Filter filter;
   @Nullable
   private final QueryMetrics<? extends Query> metrics;
-  private final long minDataTimestamp;
-  private final long maxDataTimestamp;
+  private final List<OrderBy> ordering;
   private final boolean descending;
+  private final QueryContext queryContext;
+  private final int vectorSize;
+  private final Supplier<CursorResources> resourcesSupplier;
 
-  public QueryableIndexCursorSequenceBuilder(
+  public QueryableIndexCursorHolder(
       QueryableIndex index,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      @Nullable Filter filter,
-      @Nullable QueryMetrics<? extends Query> metrics,
-      long minDataTimestamp,
-      long maxDataTimestamp,
-      boolean descending
+      CursorBuildSpec cursorBuildSpec
   )
   {
     this.index = index;
-    this.interval = interval;
-    this.virtualColumns = virtualColumns;
-    this.filter = filter;
-    this.metrics = metrics;
-    this.minDataTimestamp = minDataTimestamp;
-    this.maxDataTimestamp = maxDataTimestamp;
-    this.descending = descending;
+    this.interval = cursorBuildSpec.getInterval();
+    this.virtualColumns = cursorBuildSpec.getVirtualColumns();
+    this.aggregatorFactories = cursorBuildSpec.getAggregators();
+    this.filter = cursorBuildSpec.getFilter();
+    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    if (Cursors.preferDescendingTimeOrdering(cursorBuildSpec)) {
+      this.ordering = Collections.singletonList(OrderBy.descending(ColumnHolder.TIME_COLUMN_NAME));
+      this.descending = true;
+    } else {
+      this.ordering = Collections.singletonList(OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME));
+      this.descending = false;
+    }
+    this.queryContext = cursorBuildSpec.getQueryContext();
+    this.vectorSize = cursorBuildSpec.getQueryContext().getVectorSize();
+    this.metrics = cursorBuildSpec.getQueryMetrics();
+    this.resourcesSupplier = Suppliers.memoize(() -> new CursorResources(index, virtualColumns, filter, metrics));
   }
 
-  public Sequence<Cursor> build(final Granularity gran)
+  @Override
+  public boolean canVectorize()
   {
-    final Closer closer = Closer.create();
+    final ColumnInspector inspector = virtualColumns.wrapInspector(index);
+    if (!virtualColumns.isEmpty()) {
+      if (!queryContext.getVectorizeVirtualColumns().shouldVectorize(virtualColumns.canVectorize(inspector))) {
+        return false;
+      }
+    }
+    if (aggregatorFactories != null) {
+      for (AggregatorFactory factory : aggregatorFactories) {
+        if (!factory.canVectorize(inspector)) {
+          return false;
+        }
+      }
+    }
 
-    // Column caches shared amongst all cursors in this sequence.
-    final ColumnCache columnCache = new ColumnCache(index, closer);
+    final CursorResources resources = resourcesSupplier.get();
+    final FilterBundle filterBundle = resources.filterBundle;
+    if (filterBundle != null) {
+      if (!filterBundle.canVectorizeMatcher()) {
+        return false;
+      }
+    }
 
+    // vector cursors can't iterate backwards yet
+    return !descending;
+  }
+
+  @Override
+  public Cursor asCursor()
+  {
+    if (metrics != null) {
+      metrics.vectorized(false);
+    }
     final Offset baseOffset;
 
-    final ColumnSelectorColumnIndexSelector bitmapIndexSelector = new ColumnSelectorColumnIndexSelector(
-        index.getBitmapFactoryForDimensions(),
-        virtualColumns,
-        columnCache
-    );
-
-    final int numRows = index.getNumRows();
-    final FilterBundle filterBundle = makeFilterBundle(bitmapIndexSelector, numRows);
+    final CursorResources resources = resourcesSupplier.get();
+    final FilterBundle filterBundle = resources.filterBundle;
+    final int numRows = resources.numRows;
+    final long minDataTimestamp = resources.minDataTimestamp;
+    final long maxDataTimestamp = resources.maxDataTimestamp;
+    final NumericColumn timestamps = resources.timestamps;
+    final ColumnCache columnCache = resources.columnCache;
 
     // filterBundle will only be null if the filter itself is null, otherwise check to see if the filter
     // can use an index
@@ -121,211 +157,147 @@ public class QueryableIndexCursorSequenceBuilder
       baseOffset = BitmapOffset.of(filterBundle.getIndex().getBitmap(), descending, index.getNumRows());
     }
 
-    final NumericColumn timestamps = (NumericColumn) columnCache.getColumn(ColumnHolder.TIME_COLUMN_NAME);
+    final long timeStart = Math.max(interval.getStartMillis(), minDataTimestamp);
+    final long timeEnd = interval.getEndMillis();
 
-    Iterable<Interval> iterable = gran.getIterable(interval);
     if (descending) {
-      iterable = Lists.reverse(ImmutableList.copyOf(iterable));
-    }
-
-    return Sequences.withBaggage(
-        Sequences.map(
-            Sequences.simple(iterable),
-            new Function<Interval, Cursor>()
-            {
-              @Override
-              public Cursor apply(final Interval inputInterval)
-              {
-                final long timeStart = Math.max(interval.getStartMillis(), inputInterval.getStartMillis());
-                final long timeEnd = Math.min(
-                    interval.getEndMillis(),
-                    gran.increment(inputInterval.getStartMillis())
-                );
-
-                if (descending) {
-                  for (; baseOffset.withinBounds(); baseOffset.increment()) {
-                    if (timestamps.getLongSingleValueRow(baseOffset.getOffset()) < timeEnd) {
-                      break;
-                    }
-                  }
-                } else {
-                  for (; baseOffset.withinBounds(); baseOffset.increment()) {
-                    if (timestamps.getLongSingleValueRow(baseOffset.getOffset()) >= timeStart) {
-                      break;
-                    }
-                  }
-                }
-
-                final Offset offset = descending ?
-                                      new DescendingTimestampCheckingOffset(
-                                          baseOffset,
-                                          timestamps,
-                                          timeStart,
-                                          minDataTimestamp >= timeStart
-                                      ) :
-                                      new AscendingTimestampCheckingOffset(
-                                          baseOffset,
-                                          timestamps,
-                                          timeEnd,
-                                          maxDataTimestamp < timeEnd
-                                      );
-
-
-                final Offset baseCursorOffset = offset.clone();
-                final ColumnSelectorFactory columnSelectorFactory = new QueryableIndexColumnSelectorFactory(
-                    virtualColumns,
-                    descending,
-                    baseCursorOffset.getBaseReadableOffset(),
-                    columnCache
-                );
-                final DateTime myBucket = gran.toDateTime(inputInterval.getStartMillis());
-                // filterBundle will only be null if the filter itself is null, otherwise check to see if the filter
-                // needs to use a value matcher
-                if (filterBundle != null && filterBundle.getMatcherBundle() != null) {
-                  final ValueMatcher matcher = filterBundle.getMatcherBundle()
-                                                           .valueMatcher(
-                                                               columnSelectorFactory,
-                                                               baseCursorOffset,
-                                                               descending
-                                                           );
-                  final FilteredOffset filteredOffset = new FilteredOffset(baseCursorOffset, matcher);
-                  return new QueryableIndexCursor(filteredOffset, columnSelectorFactory, myBucket);
-                } else {
-                  return new QueryableIndexCursor(baseCursorOffset, columnSelectorFactory, myBucket);
-                }
-              }
-            }
-        ),
-        closer
-    );
-  }
-
-  public VectorCursor buildVectorized(final int vectorSize)
-  {
-    // Sanity check - matches QueryableIndexStorageAdapter.canVectorize
-    Preconditions.checkState(!descending, "!descending");
-
-    final Closer closer = Closer.create();
-    final ColumnCache columnCache = new ColumnCache(index, closer);
-
-    // Wrap the remainder of cursor setup in a try, so if an error is encountered while setting it up, we don't
-    // leak columns in the ColumnCache.
-    try {
-      final ColumnSelectorColumnIndexSelector bitmapIndexSelector = new ColumnSelectorColumnIndexSelector(
-          index.getBitmapFactoryForDimensions(),
-          virtualColumns,
-          columnCache
-      );
-
-      final int numRows = index.getNumRows();
-      final FilterBundle filterBundle = makeFilterBundle(bitmapIndexSelector, numRows);
-
-      NumericColumn timestamps = null;
-
-      final int startOffset;
-      final int endOffset;
-
-      if (interval.getStartMillis() > minDataTimestamp) {
-        timestamps = (NumericColumn) columnCache.getColumn(ColumnHolder.TIME_COLUMN_NAME);
-
-        startOffset = timeSearch(timestamps, interval.getStartMillis(), 0, index.getNumRows());
-      } else {
-        startOffset = 0;
-      }
-
-      if (interval.getEndMillis() <= maxDataTimestamp) {
-        if (timestamps == null) {
-          timestamps = (NumericColumn) columnCache.getColumn(ColumnHolder.TIME_COLUMN_NAME);
+      for (; baseOffset.withinBounds(); baseOffset.increment()) {
+        if (timestamps.getLongSingleValueRow(baseOffset.getOffset()) < timeEnd) {
+          break;
         }
-
-        endOffset = timeSearch(timestamps, interval.getEndMillis(), startOffset, index.getNumRows());
-      } else {
-        endOffset = index.getNumRows();
       }
-
-      // filterBundle will only be null if the filter itself is null, otherwise check to see if the filter can use
-      // an index
-      final VectorOffset baseOffset =
-          filterBundle == null || filterBundle.getIndex() == null
-          ? new NoFilterVectorOffset(vectorSize, startOffset, endOffset)
-          : new BitmapVectorOffset(vectorSize, filterBundle.getIndex().getBitmap(), startOffset, endOffset);
-
-      // baseColumnSelectorFactory using baseOffset is the column selector for filtering.
-      final VectorColumnSelectorFactory baseColumnSelectorFactory = makeVectorColumnSelectorFactoryForOffset(
-          columnCache,
-          baseOffset
-      );
-
-      // filterBundle will only be null if the filter itself is null, otherwise check to see if the filter needs to use
-      // a value matcher
-      if (filterBundle != null && filterBundle.getMatcherBundle() != null) {
-        final VectorValueMatcher vectorValueMatcher = filterBundle.getMatcherBundle()
-                                                                  .vectorMatcher(baseColumnSelectorFactory, baseOffset);
-        final VectorOffset filteredOffset = FilteredVectorOffset.create(
-            baseOffset,
-            vectorValueMatcher
-        );
-
-        // Now create the cursor and column selector that will be returned to the caller.
-        final VectorColumnSelectorFactory filteredColumnSelectorFactory = makeVectorColumnSelectorFactoryForOffset(
-            columnCache,
-            filteredOffset
-        );
-        return new QueryableIndexVectorCursor(filteredColumnSelectorFactory, filteredOffset, vectorSize, closer);
-      } else {
-        return new QueryableIndexVectorCursor(baseColumnSelectorFactory, baseOffset, vectorSize, closer);
+    } else {
+      for (; baseOffset.withinBounds(); baseOffset.increment()) {
+        if (timestamps.getLongSingleValueRow(baseOffset.getOffset()) >= timeStart) {
+          break;
+        }
       }
     }
-    catch (Throwable t) {
-      throw CloseableUtils.closeAndWrapInCatch(t, closer);
+
+    final Offset offset = descending ?
+                          new DescendingTimestampCheckingOffset(
+                              baseOffset,
+                              timestamps,
+                              timeStart,
+                              minDataTimestamp >= timeStart
+                          ) :
+                          new AscendingTimestampCheckingOffset(
+                              baseOffset,
+                              timestamps,
+                              timeEnd,
+                              maxDataTimestamp < timeEnd
+                          );
+
+
+    final Offset baseCursorOffset = offset.clone();
+    final ColumnSelectorFactory columnSelectorFactory = new QueryableIndexColumnSelectorFactory(
+        virtualColumns,
+        descending,
+        baseCursorOffset.getBaseReadableOffset(),
+        columnCache
+    );
+    // filterBundle will only be null if the filter itself is null, otherwise check to see if the filter
+    // needs to use a value matcher
+    if (filterBundle != null && filterBundle.getMatcherBundle() != null) {
+      final ValueMatcher matcher = filterBundle.getMatcherBundle()
+                                               .valueMatcher(
+                                                   columnSelectorFactory,
+                                                   baseCursorOffset,
+                                                   descending
+                                               );
+      final FilteredOffset filteredOffset = new FilteredOffset(baseCursorOffset, matcher);
+      return new QueryableIndexCursor(filteredOffset, columnSelectorFactory);
+    } else {
+      return new QueryableIndexCursor(baseCursorOffset, columnSelectorFactory);
     }
   }
 
   @Nullable
-  private FilterBundle makeFilterBundle(
-      ColumnSelectorColumnIndexSelector bitmapIndexSelector,
-      int numRows
-  )
+  @Override
+  public VectorCursor asVectorCursor()
   {
-    final BitmapFactory bitmapFactory = bitmapIndexSelector.getBitmapFactory();
-    final BitmapResultFactory<?> bitmapResultFactory;
+    final CursorResources resources = resourcesSupplier.get();
+    final FilterBundle filterBundle = resources.filterBundle;
+    final long minDataTimestamp = resources.minDataTimestamp;
+    final long maxDataTimestamp = resources.maxDataTimestamp;
+    final NumericColumn timestamps = resources.timestamps;
+    final ColumnCache columnCache = resources.columnCache;
+    // Wrap the remainder of cursor setup in a try, so if an error is encountered while setting it up, we don't
+    // leak columns in the ColumnCache.
+
+    // sanity check
+    if (!canVectorize()) {
+      close();
+      throw new IllegalStateException("canVectorize()");
+    }
     if (metrics != null) {
-      bitmapResultFactory = metrics.makeBitmapResultFactory(bitmapFactory);
-      metrics.reportSegmentRows(numRows);
+      metrics.vectorized(true);
+    }
+
+
+    final int startOffset;
+    final int endOffset;
+
+    if (interval.getStartMillis() > minDataTimestamp) {
+      startOffset = timeSearch(timestamps, interval.getStartMillis(), 0, index.getNumRows());
     } else {
-      bitmapResultFactory = new DefaultBitmapResultFactory(bitmapFactory);
+      startOffset = 0;
     }
-    if (filter == null) {
-      return null;
+
+    if (interval.getEndMillis() <= maxDataTimestamp) {
+      endOffset = timeSearch(timestamps, interval.getEndMillis(), startOffset, index.getNumRows());
+    } else {
+      endOffset = index.getNumRows();
     }
-    final long bitmapConstructionStartNs = System.nanoTime();
-    final FilterBundle filterBundle = filter.makeFilterBundle(
-        bitmapIndexSelector,
-        bitmapResultFactory,
-        numRows,
-        numRows,
-        false
+
+    // filterBundle will only be null if the filter itself is null, otherwise check to see if the filter can use
+    // an index
+    final VectorOffset baseOffset =
+        filterBundle == null || filterBundle.getIndex() == null
+        ? new NoFilterVectorOffset(vectorSize, startOffset, endOffset)
+        : new BitmapVectorOffset(vectorSize, filterBundle.getIndex().getBitmap(), startOffset, endOffset);
+
+    // baseColumnSelectorFactory using baseOffset is the column selector for filtering.
+    final VectorColumnSelectorFactory baseColumnSelectorFactory = makeVectorColumnSelectorFactoryForOffset(
+        columnCache,
+        baseOffset
     );
-    if (metrics != null) {
-      final long buildTime = System.nanoTime() - bitmapConstructionStartNs;
-      metrics.reportBitmapConstructionTime(buildTime);
-      final FilterBundle.BundleInfo info = filterBundle.getInfo();
-      metrics.filterBundle(info);
-      log.debug("Filter partitioning (%sms):%s", TimeUnit.NANOSECONDS.toMillis(buildTime), info);
-      if (filterBundle.getIndex() != null) {
-        metrics.reportPreFilteredRows(filterBundle.getIndex().getBitmap().size());
-      } else {
-        metrics.reportPreFilteredRows(0);
-      }
-    } else if (log.isDebugEnabled()) {
-      final FilterBundle.BundleInfo info = filterBundle.getInfo();
-      final long buildTime = System.nanoTime() - bitmapConstructionStartNs;
-      log.debug("Filter partitioning (%sms):%s", TimeUnit.NANOSECONDS.toMillis(buildTime), info);
+
+    // filterBundle will only be null if the filter itself is null, otherwise check to see if the filter needs to use
+    // a value matcher
+    if (filterBundle != null && filterBundle.getMatcherBundle() != null) {
+      final VectorValueMatcher vectorValueMatcher = filterBundle.getMatcherBundle()
+                                                                .vectorMatcher(baseColumnSelectorFactory, baseOffset);
+      final VectorOffset filteredOffset = FilteredVectorOffset.create(
+          baseOffset,
+          vectorValueMatcher
+      );
+
+      // Now create the cursor and column selector that will be returned to the caller.
+      final VectorColumnSelectorFactory filteredColumnSelectorFactory = makeVectorColumnSelectorFactoryForOffset(
+          columnCache,
+          filteredOffset
+      );
+      return new QueryableIndexVectorCursor(filteredColumnSelectorFactory, filteredOffset, vectorSize);
+    } else {
+      return new QueryableIndexVectorCursor(baseColumnSelectorFactory, baseOffset, vectorSize);
     }
-    return filterBundle;
   }
 
-  VectorColumnSelectorFactory makeVectorColumnSelectorFactoryForOffset(
+  @Override
+  public List<OrderBy> getOrdering()
+  {
+    return ordering;
+  }
+
+  @Override
+  public void close()
+  {
+    CloseableUtils.closeAndWrapExceptions(resourcesSupplier.get());
+  }
+
+
+  private VectorColumnSelectorFactory makeVectorColumnSelectorFactoryForOffset(
       ColumnCache columnCache,
       VectorOffset baseOffset
   )
@@ -391,7 +363,6 @@ public class QueryableIndexCursorSequenceBuilder
 
   private static class QueryableIndexVectorCursor implements VectorCursor
   {
-    private final Closer closer;
     private final int vectorSize;
     private final VectorOffset offset;
     private final VectorColumnSelectorFactory columnSelectorFactory;
@@ -399,14 +370,12 @@ public class QueryableIndexCursorSequenceBuilder
     public QueryableIndexVectorCursor(
         final VectorColumnSelectorFactory vectorColumnSelectorFactory,
         final VectorOffset offset,
-        final int vectorSize,
-        final Closer closer
+        final int vectorSize
     )
     {
       this.columnSelectorFactory = vectorColumnSelectorFactory;
       this.vectorSize = vectorSize;
       this.offset = offset;
-      this.closer = closer;
     }
 
     @Override
@@ -445,30 +414,17 @@ public class QueryableIndexCursorSequenceBuilder
     {
       offset.reset();
     }
-
-    @Override
-    public void close()
-    {
-      try {
-        closer.close();
-      }
-      catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-    }
   }
 
   private static class QueryableIndexCursor implements HistoricalCursor
   {
     private final Offset cursorOffset;
     private final ColumnSelectorFactory columnSelectorFactory;
-    private final DateTime bucketStart;
 
-    QueryableIndexCursor(Offset cursorOffset, ColumnSelectorFactory columnSelectorFactory, DateTime bucketStart)
+    QueryableIndexCursor(Offset cursorOffset, ColumnSelectorFactory columnSelectorFactory)
     {
       this.cursorOffset = cursorOffset;
       this.columnSelectorFactory = columnSelectorFactory;
-      this.bucketStart = bucketStart;
     }
 
     @Override
@@ -483,19 +439,13 @@ public class QueryableIndexCursorSequenceBuilder
       return columnSelectorFactory;
     }
 
-    @Override
-    public DateTime getTime()
-    {
-      return bucketStart;
-    }
-
     @Override
     public void advance()
     {
       cursorOffset.increment();
       // Must call BaseQuery.checkInterrupted() after cursorOffset.increment(), not before, because
       // FilteredOffset.increment() is a potentially long, not an "instant" operation (unlike to all other subclasses
-      // of Offset) and it returns early on interruption, leaving itself in an illegal state. We should not let
+      // of Offset) and it returns early on interruption, leaving itself in an illegal state.  We should not let
       // aggregators, etc. access this illegal state and throw a QueryInterruptedException by calling
       // BaseQuery.checkInterrupted().
       BaseQuery.checkInterrupted();
@@ -526,7 +476,6 @@ public class QueryableIndexCursorSequenceBuilder
     }
   }
 
-
   public abstract static class TimestampCheckingOffset extends Offset
   {
     final Offset baseOffset;
@@ -668,4 +617,98 @@ public class QueryableIndexCursorSequenceBuilder
       return new DescendingTimestampCheckingOffset(baseOffset.clone(), timestamps, timeLimit, allWithinThreshold);
     }
   }
+
+  private static final class CursorResources implements Closeable
+  {
+    private final Closer closer;
+    private final long minDataTimestamp;
+    private final long maxDataTimestamp;
+    private final int numRows;
+    @Nullable
+    private final Filter filter;
+    @Nullable
+    private final FilterBundle filterBundle;
+    private final NumericColumn timestamps;
+    private final ColumnCache columnCache;
+    @Nullable
+    private final QueryMetrics<? extends Query<?>> metrics;
+
+    private CursorResources(
+        QueryableIndex index,
+        VirtualColumns virtualColumns,
+        @Nullable Filter filter,
+        @Nullable QueryMetrics<? extends Query<?>> metrics
+    )
+    {
+      this.closer = Closer.create();
+      this.filter = filter;
+      this.metrics = metrics;
+      this.columnCache = new ColumnCache(index, closer);
+      final ColumnSelectorColumnIndexSelector bitmapIndexSelector = new ColumnSelectorColumnIndexSelector(
+          index.getBitmapFactoryForDimensions(),
+          virtualColumns,
+          columnCache
+      );
+      try {
+        this.numRows = index.getNumRows();
+        this.filterBundle = makeFilterBundle(bitmapIndexSelector, numRows);
+        this.timestamps = (NumericColumn) columnCache.getColumn(ColumnHolder.TIME_COLUMN_NAME);
+        this.minDataTimestamp = DateTimes.utc(timestamps.getLongSingleValueRow(0)).getMillis();
+        this.maxDataTimestamp = DateTimes.utc(timestamps.getLongSingleValueRow(timestamps.length() - 1)).getMillis();
+      }
+      catch (Throwable t) {
+        throw CloseableUtils.closeAndWrapInCatch(t, closer);
+      }
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+      closer.close();
+    }
+
+    @Nullable
+    private FilterBundle makeFilterBundle(
+        ColumnSelectorColumnIndexSelector bitmapIndexSelector,
+        int numRows
+    )
+    {
+      final BitmapFactory bitmapFactory = bitmapIndexSelector.getBitmapFactory();
+      final BitmapResultFactory<?> bitmapResultFactory;
+      if (metrics != null) {
+        bitmapResultFactory = metrics.makeBitmapResultFactory(bitmapFactory);
+        metrics.reportSegmentRows(numRows);
+      } else {
+        bitmapResultFactory = new DefaultBitmapResultFactory(bitmapFactory);
+      }
+      if (filter == null) {
+        return null;
+      }
+      final long bitmapConstructionStartNs = System.nanoTime();
+      final FilterBundle filterBundle = filter.makeFilterBundle(
+          bitmapIndexSelector,
+          bitmapResultFactory,
+          numRows,
+          numRows,
+          false
+      );
+      if (metrics != null) {
+        final long buildTime = System.nanoTime() - bitmapConstructionStartNs;
+        metrics.reportBitmapConstructionTime(buildTime);
+        final FilterBundle.BundleInfo info = filterBundle.getInfo();
+        metrics.filterBundle(info);
+        log.debug("Filter partitioning (%sms):%s", TimeUnit.NANOSECONDS.toMillis(buildTime), info);
+        if (filterBundle.getIndex() != null) {
+          metrics.reportPreFilteredRows(filterBundle.getIndex().getBitmap().size());
+        } else {
+          metrics.reportPreFilteredRows(0);
+        }
+      } else if (log.isDebugEnabled()) {
+        final FilterBundle.BundleInfo info = filterBundle.getInfo();
+        final long buildTime = System.nanoTime() - bitmapConstructionStartNs;
+        log.debug("Filter partitioning (%sms):%s", TimeUnit.NANOSECONDS.toMillis(buildTime), info);
+      }
+      return filterBundle;
+    }
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
index 57c7da95360..7d860465cb6 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
@@ -20,13 +20,6 @@
 package org.apache.druid.segment;
 
 import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.column.BaseColumn;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
@@ -35,7 +28,6 @@ import org.apache.druid.segment.column.DictionaryEncodedColumn;
 import org.apache.druid.segment.column.NumericColumn;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex;
-import org.apache.druid.segment.vector.VectorCursor;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -43,7 +35,6 @@ import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.LinkedHashSet;
-import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -179,97 +170,11 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
   }
 
   @Override
-  public boolean canVectorize(
-      @Nullable final Filter filter,
-      final VirtualColumns virtualColumns,
-      final boolean descending
-  )
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    if (filter != null) {
-      // ideally we would allow stuff to vectorize if we can build indexes even if the value matcher cannot be
-      // vectorized, this used to be true in fact, but changes to filter partitioning (FilterBundle) have caused
-      // the only way to know this to be building the bitmaps since BitmapColumnIndex can return null.
-      // this will be changed in a future refactor of cursor building, at which point this method can just return
-      // true if !descending...
-      final boolean filterCanVectorize = filter.canVectorizeMatcher(this);
-
-      if (!filterCanVectorize) {
-        return false;
-      }
-    }
-
-    // vector cursors can't iterate backwards yet
-    return !descending;
-  }
-
-  @Override
-  @Nullable
-  public VectorCursor makeVectorCursor(
-      @Nullable final Filter filter,
-      final Interval interval,
-      final VirtualColumns virtualColumns,
-      final boolean descending,
-      final int vectorSize,
-      @Nullable final QueryMetrics<?> queryMetrics
-  )
-  {
-    if (!canVectorize(filter, virtualColumns, descending)) {
-      throw new ISE("Cannot vectorize. Check 'canVectorize' before calling 'makeVectorCursor'.");
-    }
-
-    if (queryMetrics != null) {
-      queryMetrics.vectorized(true);
-    }
-
-    final Interval actualInterval = computeCursorInterval(Granularities.ALL, interval);
-
-    if (actualInterval == null) {
-      return null;
-    }
-    return new QueryableIndexCursorSequenceBuilder(
+    return new QueryableIndexCursorHolder(
         index,
-        actualInterval,
-        virtualColumns,
-        filter,
-        queryMetrics,
-        getMinTime().getMillis(),
-        getMaxTime().getMillis(),
-        descending
-    ).buildVectorized(vectorSize > 0 ? vectorSize : DEFAULT_VECTOR_SIZE);
-  }
-
-  @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      Granularity gran,
-      boolean descending,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
-  {
-    if (queryMetrics != null) {
-      queryMetrics.vectorized(false);
-    }
-
-    final Interval actualInterval = computeCursorInterval(gran, interval);
-
-    if (actualInterval == null) {
-      return Sequences.empty();
-    }
-
-    return Sequences.filter(
-        new QueryableIndexCursorSequenceBuilder(
-            index,
-            actualInterval,
-            virtualColumns,
-            filter,
-            queryMetrics,
-            getMinTime().getMillis(),
-            getMaxTime().getMillis(),
-            descending
-        ).build(gran),
-        Objects::nonNull
+        CursorBuildSpec.builder(spec).build()
     );
   }
 
@@ -288,18 +193,4 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
       this.maxTime = DateTimes.utc(column.getLongSingleValueRow(column.length() - 1));
     }
   }
-
-  @Nullable
-  private Interval computeCursorInterval(final Granularity gran, final Interval interval)
-  {
-    final DateTime minTime = getMinTime();
-    final DateTime maxTime = getMaxTime();
-    final Interval dataInterval = new Interval(minTime, gran.bucketEnd(maxTime));
-
-    if (!interval.overlaps(dataInterval)) {
-      return null;
-    }
-
-    return interval.overlap(dataInterval);
-  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/RowBasedCursor.java b/processing/src/main/java/org/apache/druid/segment/RowBasedCursor.java
index f843fe63131..d1a70783f6b 100644
--- a/processing/src/main/java/org/apache/druid/segment/RowBasedCursor.java
+++ b/processing/src/main/java/org/apache/druid/segment/RowBasedCursor.java
@@ -19,7 +19,6 @@
 
 package org.apache.druid.segment;
 
-import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.ValueMatcher;
@@ -42,7 +41,7 @@ public class RowBasedCursor<RowType> implements Cursor
   private final ToLongFunction<RowType> timestampFunction;
   private final Interval interval;
   private final boolean descending;
-  private final DateTime cursorTime;
+  private final DateTime startTime;
   private final ColumnSelectorFactory columnSelectorFactory;
   private final ValueMatcher valueMatcher;
 
@@ -54,7 +53,6 @@ public class RowBasedCursor<RowType> implements Cursor
       @Nullable final Filter filter,
       final Interval interval,
       final VirtualColumns virtualColumns,
-      final Granularity gran,
       final boolean descending,
       final RowSignature rowSignature
   )
@@ -63,7 +61,7 @@ public class RowBasedCursor<RowType> implements Cursor
     this.timestampFunction = rowAdapter.timestampFunction();
     this.interval = interval;
     this.descending = descending;
-    this.cursorTime = gran.toDateTime(interval.getStartMillis());
+    this.startTime = descending ? interval.getEnd().minus(1) : interval.getStart();
     this.columnSelectorFactory = virtualColumns.wrap(
         new RowBasedColumnSelectorFactory<>(
             rowWalker::currentRow,
@@ -80,8 +78,7 @@ public class RowBasedCursor<RowType> implements Cursor
     } else {
       this.valueMatcher = filter.makeMatcher(this.columnSelectorFactory);
     }
-
-    rowWalker.skipToDateTime(descending ? interval.getEnd().minus(1) : interval.getStart(), descending);
+    rowWalker.skipToDateTime(startTime, descending);
     advanceToMatchingRow();
   }
 
@@ -91,12 +88,6 @@ public class RowBasedCursor<RowType> implements Cursor
     return columnSelectorFactory;
   }
 
-  @Override
-  public DateTime getTime()
-  {
-    return cursorTime;
-  }
-
   @Override
   public void advance()
   {
@@ -129,7 +120,7 @@ public class RowBasedCursor<RowType> implements Cursor
   {
     rowId = 0;
     rowWalker.reset();
-    rowWalker.skipToDateTime(descending ? interval.getEnd().minus(1) : interval.getStart(), descending);
+    rowWalker.skipToDateTime(startTime, descending);
     advanceToMatchingRow();
   }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/RowBasedSegment.java b/processing/src/main/java/org/apache/druid/segment/RowBasedSegment.java
index d9b2cfbd324..26ff6594bbb 100644
--- a/processing/src/main/java/org/apache/druid/segment/RowBasedSegment.java
+++ b/processing/src/main/java/org/apache/druid/segment/RowBasedSegment.java
@@ -40,8 +40,8 @@ public class RowBasedSegment<RowType> implements Segment
    * Create a row-based segment.
    *
    * The provided "rowIterable" must be in time-order according to the provided {@link RowAdapter#timestampFunction()}.
-   * The cursor returned by {@link RowBasedStorageAdapter#makeCursors} makes no attempt to verify this, and callers
-   * will expect it.
+   * The cursor returned by {@link RowBasedStorageAdapter#makeCursorHolder(CursorBuildSpec)} makes no attempt to verify
+   * this, and callers will expect it.
    *
    * The provided "rowSignature" will be used for reporting available columns and their capabilities to users of
    * {@link #asStorageAdapter()}. Note that the {@link ColumnSelectorFactory} implementation returned by this segment's
diff --git a/processing/src/main/java/org/apache/druid/segment/RowBasedStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/RowBasedStorageAdapter.java
index efd3b96829d..f4330709a44 100644
--- a/processing/src/main/java/org/apache/druid/segment/RowBasedStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/RowBasedStorageAdapter.java
@@ -20,21 +20,20 @@
 package org.apache.druid.segment;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
-import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.guava.SimpleSequence;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
+import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.ListIndexed;
+import org.apache.druid.utils.CloseableUtils;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -168,52 +167,52 @@ public class RowBasedStorageAdapter<RowType> implements StorageAdapter
   }
 
   @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable final Filter filter,
-      final Interval queryInterval,
-      final VirtualColumns virtualColumns,
-      final Granularity gran,
-      final boolean descending,
-      @Nullable final QueryMetrics<?> queryMetrics
-  )
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    final Interval actualInterval = queryInterval.overlap(new Interval(getMinTime(), gran.bucketEnd(getMaxTime())));
-
-    if (actualInterval == null) {
-      return Sequences.empty();
+    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    final List<OrderBy> ordering;
+    final boolean descending;
+    if (Cursors.preferDescendingTimeOrdering(spec)) {
+      ordering = Cursors.descendingTimeOrder();
+      descending = true;
+    } else {
+      ordering = Cursors.ascendingTimeOrder();
+      descending = false;
     }
+    return new CursorHolder()
+    {
+      final Closer closer = Closer.create();
 
-    if (!isQueryGranularityAllowed(actualInterval, gran)) {
-      throw new IAE(
-          "Cannot support interval [%s] with granularity [%s]",
-          Intervals.ETERNITY.equals(actualInterval) ? "ETERNITY" : actualInterval,
-          gran
-      );
-    }
+      @Override
+      public Cursor asCursor()
+      {
+        final RowWalker<RowType> rowWalker = closer.register(
+            new RowWalker<>(descending ? reverse(rowSequence) : rowSequence, rowAdapter)
+        );
+        return new RowBasedCursor<>(
+            rowWalker,
+            rowAdapter,
+            spec.getFilter(),
+            spec.getInterval(),
+            spec.getVirtualColumns(),
+            descending,
+            rowSignature
+        );
+      }
 
-    final RowWalker<RowType> rowWalker = new RowWalker<>(
-        descending ? reverse(rowSequence) : rowSequence,
-        rowAdapter
-    );
+      @Nullable
+      @Override
+      public List<OrderBy> getOrdering()
+      {
+        return ordering;
+      }
 
-    final Iterable<Interval> bucketIntervals = gran.getIterable(actualInterval);
-
-    return Sequences.simple(
-        Iterables.transform(
-            descending ? reverse(bucketIntervals) : bucketIntervals,
-            bucketInterval ->
-                (Cursor) new RowBasedCursor<>(
-                    rowWalker,
-                    rowAdapter,
-                    filter,
-                    bucketInterval,
-                    virtualColumns,
-                    gran,
-                    descending,
-                    rowSignature
-                )
-        )
-    ).withBaggage(rowWalker::close);
+      @Override
+      public void close()
+      {
+        CloseableUtils.closeAndWrapExceptions(closer);
+      }
+    };
   }
 
   /**
diff --git a/processing/src/main/java/org/apache/druid/segment/SegmentWrangler.java b/processing/src/main/java/org/apache/druid/segment/SegmentWrangler.java
index ec08fd86527..4e39ca00f18 100644
--- a/processing/src/main/java/org/apache/druid/segment/SegmentWrangler.java
+++ b/processing/src/main/java/org/apache/druid/segment/SegmentWrangler.java
@@ -39,7 +39,8 @@ public interface SegmentWrangler
    *
    * @return Segments that, collectively, contain data for dataSource. May be empty if dataSource does not exist or
    * has no data in the provided intervals. May contain data outside the provided intervals, so callers should
-   * filter it down further, e.g. through the "interval" parameter of {@link StorageAdapter#makeCursors}
+   * filter it down further, e.g. through the "interval" parameter of {@link CursorBuildSpec} for
+   * {@link StorageAdapter#makeCursorHolder(CursorBuildSpec)}
    */
   Iterable<Segment> getSegmentsForIntervals(DataSource dataSource, Iterable<Interval> intervals);
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
index 2d3fc6a50bb..fba947f73d2 100644
--- a/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
@@ -21,20 +21,99 @@ package org.apache.druid.segment;
 
 import com.google.common.collect.Iterables;
 import org.apache.druid.guice.annotations.PublicApi;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
+import org.apache.druid.segment.vector.VectorCursor;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
+import java.util.List;
 import java.util.Optional;
 
 /**
  */
 @PublicApi
-public interface StorageAdapter extends CursorFactory, ColumnInspector
+public interface StorageAdapter extends CursorFactory, ColumnInspector, CursorHolderFactory
 {
+
+  /**
+   * Build a {@link CursorHolder} which can provide {@link Cursor} and {@link VectorCursor} (if capable) which allows
+   * scanning segments and creating {@link ColumnSelectorFactory} and
+   * {@link org.apache.druid.segment.vector.VectorColumnSelectorFactory} respectively to read row values at the cursor
+   * position.
+   */
+  @Override
+  default CursorHolder makeCursorHolder(CursorBuildSpec spec)
+  {
+    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    final List<OrderBy> ordering;
+    final boolean descending;
+    if (Cursors.preferDescendingTimeOrdering(spec)) {
+      ordering = Cursors.descendingTimeOrder();
+      descending = true;
+    } else {
+      ordering = Cursors.ascendingTimeOrder();
+      descending = false;
+    }
+    return new CursorHolder()
+    {
+      @Override
+      public boolean canVectorize()
+      {
+        return StorageAdapter.this.canVectorize(
+            spec.getFilter(),
+            spec.getVirtualColumns(),
+            Cursors.preferDescendingTimeOrdering(spec)
+        );
+      }
+
+      @Override
+      public Cursor asCursor()
+      {
+        return Iterables.getOnlyElement(
+            StorageAdapter.this.makeCursors(
+                spec.getFilter(),
+                spec.getInterval(),
+                spec.getVirtualColumns(),
+                Granularities.ALL,
+                descending,
+                spec.getQueryMetrics()
+            ).toList()
+        );
+      }
+
+      @Override
+      public VectorCursor asVectorCursor()
+      {
+        return StorageAdapter.this.makeVectorCursor(
+            spec.getFilter(),
+            spec.getInterval(),
+            spec.getVirtualColumns(),
+            descending,
+            spec.getQueryContext().getVectorSize(),
+            spec.getQueryMetrics()
+        );
+      }
+
+      @Nullable
+      @Override
+      public List<OrderBy> getOrdering()
+      {
+        return ordering;
+      }
+
+      @Override
+      public void close()
+      {
+        // consuming sequences of CursorFactory are expected to close themselves.
+      }
+    };
+  }
+
   Interval getInterval();
   Indexed<String> getAvailableDimensions();
   Iterable<String> getAvailableMetrics();
diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java b/processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java
index 5404937f519..a2a920cba1d 100644
--- a/processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java
+++ b/processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java
@@ -24,7 +24,7 @@ import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.column.ColumnCapabilities;
-import org.joda.time.DateTime;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 import javax.annotation.Nullable;
 import java.util.Arrays;
@@ -62,6 +62,7 @@ public class UnnestColumnValueSelectorCursor implements Cursor
   private final String outputName;
   private int index;
   private Object currentVal;
+  @MonotonicNonNull
   private List<Object> unnestListForCurrentRow;
   private boolean needInitialization;
 
@@ -210,12 +211,6 @@ public class UnnestColumnValueSelectorCursor implements Cursor
     };
   }
 
-  @Override
-  public DateTime getTime()
-  {
-    return baseCursor.getTime();
-  }
-
   @Override
   public void advance()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java b/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java
index 4d4aeaf7046..398c0a21def 100644
--- a/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java
+++ b/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java
@@ -29,7 +29,6 @@ import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.data.IndexedInts;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
-import org.joda.time.DateTime;
 
 import javax.annotation.Nullable;
 
@@ -74,6 +73,7 @@ public class UnnestDimensionCursor implements Cursor
   @MonotonicNonNull
   private IndexedInts indexedIntsForCurrentRow;
   private boolean needInitialization;
+  @MonotonicNonNull
   private SingleIndexInts indexIntsForRow;
   private final int nullId;
   private final int idOffset;
@@ -259,12 +259,6 @@ public class UnnestDimensionCursor implements Cursor
     };
   }
 
-  @Override
-  public DateTime getTime()
-  {
-    return baseCursor.getTime();
-  }
-
   @Override
   public void advance()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
index 752c6f48678..a9f215892d1 100644
--- a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
@@ -20,13 +20,12 @@
 package org.apache.druid.segment;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
-import org.apache.druid.query.QueryMetrics;
+import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.filter.BooleanFilter;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.filter.EqualityFilter;
@@ -49,6 +48,7 @@ import org.apache.druid.segment.filter.OrFilter;
 import org.apache.druid.segment.filter.SelectorFilter;
 import org.apache.druid.segment.join.PostJoinCursor;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
+import org.apache.druid.utils.CloseableUtils;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -57,8 +57,8 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Objects;
 import java.util.Set;
+import java.util.function.Supplier;
 
 /**
  * This class serves as the Storage Adapter for the Unnest Segment and is responsible for creating the cursors
@@ -92,65 +92,77 @@ public class UnnestStorageAdapter implements StorageAdapter
   }
 
   @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      Granularity gran,
-      boolean descending,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    final String inputColumn = getUnnestInputIfDirectAccess(unnestColumn);
+    final String input = getUnnestInputIfDirectAccess(unnestColumn);
     final Pair<Filter, Filter> filterPair = computeBaseAndPostUnnestFilters(
-        filter,
+        spec.getFilter(),
         unnestFilter != null ? unnestFilter.toFilter() : null,
-        virtualColumns,
-        inputColumn,
-        inputColumn == null ? null : virtualColumns.getColumnCapabilitiesWithFallback(baseAdapter, inputColumn)
+        spec.getVirtualColumns(),
+        input,
+        input == null ? null : spec.getVirtualColumns().getColumnCapabilitiesWithFallback(baseAdapter, input)
     );
+    final CursorBuildSpec unnestBuildSpec =
+        CursorBuildSpec.builder(spec)
+                       .setFilter(filterPair.lhs)
+                       .setVirtualColumns(VirtualColumns.create(Collections.singletonList(unnestColumn)))
+                       .build();
 
-    final Sequence<Cursor> baseCursorSequence = baseAdapter.makeCursors(
-        filterPair.lhs,
-        interval,
-        VirtualColumns.create(Collections.singletonList(unnestColumn)),
-        gran,
-        descending,
-        queryMetrics
-    );
+    return new CursorHolder()
+    {
+      final Closer closer = Closer.create();
+      final Supplier<CursorHolder> cursorHolderSupplier = Suppliers.memoize(
+          () -> closer.register(baseAdapter.makeCursorHolder(unnestBuildSpec))
+      );
 
-    return Sequences.map(
-        baseCursorSequence,
-        cursor -> {
-          Objects.requireNonNull(cursor);
-          final ColumnCapabilities capabilities = unnestColumn.capabilities(
+      @Override
+      public Cursor asCursor()
+      {
+        final Cursor cursor = cursorHolderSupplier.get().asCursor();
+        if (cursor == null) {
+          return null;
+        }
+        final ColumnCapabilities capabilities = unnestColumn.capabilities(
+            cursor.getColumnSelectorFactory(),
+            unnestColumn.getOutputName()
+        );
+        final Cursor unnestCursor;
+
+        if (useDimensionCursor(capabilities)) {
+          unnestCursor = new UnnestDimensionCursor(
+              cursor,
               cursor.getColumnSelectorFactory(),
-              unnestColumn.getOutputName()
+              unnestColumn,
+              outputColumnName
           );
-          final Cursor unnestCursor;
-
-          if (useDimensionCursor(capabilities)) {
-            unnestCursor = new UnnestDimensionCursor(
-                cursor,
-                cursor.getColumnSelectorFactory(),
-                unnestColumn,
-                outputColumnName
-            );
-          } else {
-            unnestCursor = new UnnestColumnValueSelectorCursor(
-                cursor,
-                cursor.getColumnSelectorFactory(),
-                unnestColumn,
-                outputColumnName
-            );
-          }
-          return PostJoinCursor.wrap(
-              unnestCursor,
-              virtualColumns,
-              filterPair.rhs
+        } else {
+          unnestCursor = new UnnestColumnValueSelectorCursor(
+              cursor,
+              cursor.getColumnSelectorFactory(),
+              unnestColumn,
+              outputColumnName
           );
         }
-    );
+        return PostJoinCursor.wrap(
+            unnestCursor,
+            spec.getVirtualColumns(),
+            filterPair.rhs
+        );
+      }
+
+      @Nullable
+      @Override
+      public List<OrderBy> getOrdering()
+      {
+        return cursorHolderSupplier.get().getOrdering();
+      }
+
+      @Override
+      public void close()
+      {
+        CloseableUtils.closeAndWrapExceptions(closer);
+      }
+    };
   }
 
   @Override
@@ -270,9 +282,9 @@ public class UnnestStorageAdapter implements StorageAdapter
   /**
    * Split queryFilter into pre- and post-correlate filters.
    *
-   * @param queryFilter            query filter passed to makeCursors
+   * @param queryFilter            query filter from {@link CursorBuildSpec}
    * @param unnestFilter           filter on unnested column passed to PostUnnestCursor
-   * @param queryVirtualColumns    query virtual columns passed to makeCursors
+   * @param queryVirtualColumns    query virtual columns from {@link CursorBuildSpec}
    * @param inputColumn            input column to unnest if it's a direct access; otherwise null
    * @param inputColumnCapabilites input column capabilities if known; otherwise null
    * @return pair of pre- and post-unnest filters
@@ -468,7 +480,7 @@ public class UnnestStorageAdapter implements StorageAdapter
    * while in case B, due to presence of the expression virtual column expressionVirtualColumn("j0.unnest", "array(\"dim1\",\"dim2\")", ColumnType.STRING_ARRAY)
    * the filters on d12 cannot be pushed to the pre filters
    *
-   * @param queryFilter            query filter passed to makeCursors
+   * @param queryFilter            query filter from {@link CursorBuildSpec}
    * @param inputColumn            input column to unnest if it's a direct access; otherwise null
    * @param inputColumnCapabilites input column capabilities if known; otherwise null
    */
diff --git a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
index a32a85d16c7..bc2609b4a00 100644
--- a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
+++ b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
@@ -30,7 +30,6 @@ import com.google.common.collect.Sets;
 import org.apache.druid.java.util.common.Cacheable;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.query.Query;
 import org.apache.druid.query.cache.CacheKeyBuilder;
 import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.filter.ColumnIndexSelector;
@@ -129,15 +128,6 @@ public class VirtualColumns implements Cacheable
     return virtualColumns == null ? EMPTY : virtualColumns;
   }
 
-  public static boolean shouldVectorize(Query<?> query, VirtualColumns virtualColumns, ColumnInspector inspector)
-  {
-    if (virtualColumns.getVirtualColumns().length > 0) {
-      return query.context().getVectorizeVirtualColumns().shouldVectorize(virtualColumns.canVectorize(inspector));
-    } else {
-      return true;
-    }
-  }
-
   // For equals, hashCode, toString, and serialization:
   private final List<VirtualColumn> virtualColumns;
   private final List<String> virtualColumnNames;
diff --git a/processing/src/main/java/org/apache/druid/segment/data/Offset.java b/processing/src/main/java/org/apache/druid/segment/data/Offset.java
index ea0c5e046e2..1fef6769936 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/Offset.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/Offset.java
@@ -41,9 +41,15 @@ import org.apache.druid.query.monomorphicprocessing.CalledFromHotLoop;
 @SubclassesMustBePublic
 public abstract class Offset implements ReadableOffset, Cloneable
 {
+  /**
+   * Move to next offset
+   */
   @CalledFromHotLoop
   public abstract void increment();
 
+  /**
+   * Check if current offset position is within the valid range
+   */
   @CalledFromHotLoop
   public abstract boolean withinBounds();
 
diff --git a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java
index 036c6a8250a..dfc618acad8 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java
@@ -188,6 +188,17 @@ public class AndFilter implements BooleanFilter
           }
           return makeVectorMatcher(vectorMatchers);
         }
+
+        @Override
+        public boolean canVectorize()
+        {
+          for (FilterBundle.MatcherBundle bundle : matcherBundles) {
+            if (!bundle.canVectorize()) {
+              return false;
+            }
+          }
+          return true;
+        }
       };
     } else {
       matcherBundle = null;
diff --git a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java
index 0665aabf195..700b2fbfa16 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java
@@ -243,6 +243,17 @@ public class OrFilter implements BooleanFilter
             }
             return makeVectorMatcher(matchers);
           }
+
+          @Override
+          public boolean canVectorize()
+          {
+            for (FilterBundle.MatcherBundle bundle : allMatcherBundles) {
+              if (!bundle.canVectorize()) {
+                return false;
+              }
+            }
+            return true;
+          }
         }
     );
   }
@@ -527,6 +538,12 @@ public class OrFilter implements BooleanFilter
             }
         );
       }
+
+      @Override
+      public boolean canVectorize()
+      {
+        return bundle.getMatcherBundle() == null || bundle.getMatcherBundle().canVectorize();
+      }
     };
   }
 
@@ -585,6 +602,12 @@ public class OrFilter implements BooleanFilter
       {
         return convertIndexToVectorValueMatcher(baseOffset, partialIndex);
       }
+
+      @Override
+      public boolean canVectorize()
+      {
+        return true;
+      }
     };
   }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java
index 06fc7195d52..5265f4f2c44 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java
@@ -222,7 +222,7 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
   }
 
   private final long minTimestamp;
-  private final Granularity gran;
+  private final Granularity queryGranularity;
   private final boolean rollup;
   private final List<Function<InputRow, InputRow>> rowTransformers;
   private final VirtualColumns virtualColumns;
@@ -266,7 +266,7 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
   )
   {
     this.minTimestamp = incrementalIndexSchema.getMinTimestamp();
-    this.gran = incrementalIndexSchema.getGran();
+    this.queryGranularity = incrementalIndexSchema.getQueryGranularity();
     this.rollup = incrementalIndexSchema.isRollup();
     this.virtualColumns = incrementalIndexSchema.getVirtualColumns();
     this.metrics = incrementalIndexSchema.getMetrics();
@@ -284,7 +284,7 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
         null,
         getCombiningAggregators(metrics),
         incrementalIndexSchema.getTimestampSpec(),
-        this.gran,
+        this.queryGranularity,
         this.rollup
     );
 
@@ -658,7 +658,7 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
 
     long truncated = 0;
     if (row.getTimestamp() != null) {
-      truncated = gran.bucketStart(row.getTimestampFromEpoch());
+      truncated = queryGranularity.bucketStart(row.getTimestampFromEpoch());
     }
     IncrementalIndexRow incrementalIndexRow = IncrementalIndexRow.createTimeAndDimswithDimsKeySize(
         Math.max(truncated, minTimestamp),
@@ -837,7 +837,7 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
   public Interval getInterval()
   {
     DateTime min = DateTimes.utc(minTimestamp);
-    return new Interval(min, isEmpty() ? min : gran.increment(DateTimes.utc(getMaxTimeMillis())));
+    return new Interval(min, isEmpty() ? min : queryGranularity.increment(DateTimes.utc(getMaxTimeMillis())));
   }
 
   @Nullable
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexCursorHolder.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexCursorHolder.java
new file mode 100644
index 00000000000..f0966a828eb
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexCursorHolder.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.incremental;
+
+import com.google.common.collect.Iterators;
+import org.apache.druid.query.BaseQuery;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.query.filter.Filter;
+import org.apache.druid.query.filter.ValueMatcher;
+import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
+import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.filter.ValueMatchers;
+import org.joda.time.Interval;
+
+import javax.annotation.Nullable;
+import java.util.Iterator;
+import java.util.List;
+
+public class IncrementalIndexCursorHolder implements CursorHolder
+{
+  private final IncrementalIndexStorageAdapter storageAdapter;
+  private final IncrementalIndex index;
+  private final CursorBuildSpec spec;
+  private final List<OrderBy> ordering;
+  private final boolean descending;
+
+  public IncrementalIndexCursorHolder(
+      IncrementalIndexStorageAdapter storageAdapter,
+      IncrementalIndex index,
+      CursorBuildSpec spec
+  )
+  {
+    this.storageAdapter = storageAdapter;
+    this.index = index;
+    this.spec = spec;
+    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    if (Cursors.preferDescendingTimeOrdering(spec)) {
+      this.ordering = Cursors.descendingTimeOrder();
+      this.descending = true;
+    } else {
+      this.ordering = Cursors.ascendingTimeOrder();
+      this.descending = false;
+    }
+  }
+
+  @Override
+  public Cursor asCursor()
+  {
+    if (index.isEmpty()) {
+      return null;
+    }
+
+    if (spec.getQueryMetrics() != null) {
+      spec.getQueryMetrics().vectorized(false);
+    }
+
+
+    return new IncrementalIndexCursor(
+        storageAdapter,
+        index,
+        spec.getVirtualColumns(),
+        descending,
+        spec.getFilter(),
+        spec.getInterval()
+    );
+  }
+
+  @Override
+  public List<OrderBy> getOrdering()
+  {
+    return ordering;
+  }
+
+  static class IncrementalIndexCursor implements Cursor
+  {
+    private IncrementalIndexRowHolder currEntry;
+    private final ColumnSelectorFactory columnSelectorFactory;
+    private final ValueMatcher filterMatcher;
+    private final int maxRowIndex;
+    private final IncrementalIndex.FactsHolder facts;
+    private Iterator<IncrementalIndexRow> baseIter;
+    private Iterable<IncrementalIndexRow> cursorIterable;
+    private boolean emptyRange;
+    private int numAdvanced;
+    private boolean done;
+
+    IncrementalIndexCursor(
+        IncrementalIndexStorageAdapter storageAdapter,
+        IncrementalIndex index,
+        VirtualColumns virtualColumns,
+        boolean descending,
+        @Nullable Filter filter,
+        Interval actualInterval
+    )
+    {
+      currEntry = new IncrementalIndexRowHolder();
+      columnSelectorFactory = new IncrementalIndexColumnSelectorFactory(
+          storageAdapter,
+          virtualColumns,
+          descending,
+          currEntry
+      );
+      // Set maxRowIndex before creating the filterMatcher. See https://github.com/apache/druid/pull/6340
+      maxRowIndex = index.getLastRowIndex();
+      filterMatcher = filter == null ? ValueMatchers.allTrue() : filter.makeMatcher(columnSelectorFactory);
+      numAdvanced = -1;
+      facts = index.getFacts();
+      cursorIterable = facts.timeRangeIterable(
+          descending,
+          actualInterval.getStartMillis(),
+          actualInterval.getEndMillis()
+      );
+      emptyRange = !cursorIterable.iterator().hasNext();
+
+      reset();
+    }
+
+    @Override
+    public ColumnSelectorFactory getColumnSelectorFactory()
+    {
+      return columnSelectorFactory;
+    }
+
+    @Override
+    public void advance()
+    {
+      if (!baseIter.hasNext()) {
+        done = true;
+        return;
+      }
+
+      while (baseIter.hasNext()) {
+        BaseQuery.checkInterrupted();
+
+        IncrementalIndexRow entry = baseIter.next();
+        if (beyondMaxRowIndex(entry.getRowIndex())) {
+          continue;
+        }
+
+        currEntry.set(entry);
+
+        if (filterMatcher.matches(false)) {
+          return;
+        }
+      }
+
+      done = true;
+    }
+
+    @Override
+    public void advanceUninterruptibly()
+    {
+      if (!baseIter.hasNext()) {
+        done = true;
+        return;
+      }
+
+      while (baseIter.hasNext()) {
+        if (Thread.currentThread().isInterrupted()) {
+          return;
+        }
+
+        IncrementalIndexRow entry = baseIter.next();
+        if (beyondMaxRowIndex(entry.getRowIndex())) {
+          continue;
+        }
+
+        currEntry.set(entry);
+
+        if (filterMatcher.matches(false)) {
+          return;
+        }
+      }
+
+      done = true;
+    }
+
+    @Override
+    public boolean isDone()
+    {
+      return done;
+    }
+
+    @Override
+    public boolean isDoneOrInterrupted()
+    {
+      return isDone() || Thread.currentThread().isInterrupted();
+    }
+
+    @Override
+    public void reset()
+    {
+      baseIter = cursorIterable.iterator();
+
+      if (numAdvanced == -1) {
+        numAdvanced = 0;
+      } else {
+        Iterators.advance(baseIter, numAdvanced);
+      }
+
+      BaseQuery.checkInterrupted();
+
+      boolean foundMatched = false;
+      while (baseIter.hasNext()) {
+        IncrementalIndexRow entry = baseIter.next();
+        if (beyondMaxRowIndex(entry.getRowIndex())) {
+          numAdvanced++;
+          continue;
+        }
+        currEntry.set(entry);
+        if (filterMatcher.matches(false)) {
+          foundMatched = true;
+          break;
+        }
+
+        numAdvanced++;
+      }
+
+      done = !foundMatched && (emptyRange || !baseIter.hasNext());
+    }
+
+    private boolean beyondMaxRowIndex(int rowIndex)
+    {
+      // ignore rows whose rowIndex is beyond the maxRowIndex
+      // rows are order by timestamp, not rowIndex,
+      // so we still need to go through all rows to skip rows added after cursor created
+      return rowIndex > maxRowIndex;
+    }
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRowHolder.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRowHolder.java
index c46ef559b00..f8841efae15 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRowHolder.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRowHolder.java
@@ -38,6 +38,7 @@ public class IncrementalIndexRowHolder implements LongColumnSelector
   @Nullable
   private IncrementalIndexRow currEntry = null;
 
+  @Nullable
   public IncrementalIndexRow get()
   {
     return currEntry;
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexSchema.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexSchema.java
index d127f3d89bb..8866c413dc7 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexSchema.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexSchema.java
@@ -40,7 +40,7 @@ public class IncrementalIndexSchema
 
   private final long minTimestamp;
   private final TimestampSpec timestampSpec;
-  private final Granularity gran;
+  private final Granularity queryGranularity;
   private final VirtualColumns virtualColumns;
   private final DimensionsSpec dimensionsSpec;
   private final AggregatorFactory[] metrics;
@@ -49,7 +49,7 @@ public class IncrementalIndexSchema
   public IncrementalIndexSchema(
       long minTimestamp,
       TimestampSpec timestampSpec,
-      Granularity gran,
+      Granularity queryGranularity,
       VirtualColumns virtualColumns,
       DimensionsSpec dimensionsSpec,
       AggregatorFactory[] metrics,
@@ -58,7 +58,7 @@ public class IncrementalIndexSchema
   {
     this.minTimestamp = minTimestamp;
     this.timestampSpec = timestampSpec;
-    this.gran = gran;
+    this.queryGranularity = queryGranularity;
     this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns);
     this.dimensionsSpec = dimensionsSpec;
     this.metrics = metrics;
@@ -75,9 +75,9 @@ public class IncrementalIndexSchema
     return timestampSpec;
   }
 
-  public Granularity getGran()
+  public Granularity getQueryGranularity()
   {
-    return gran;
+    return queryGranularity;
   }
 
   public VirtualColumns getVirtualColumns()
@@ -104,7 +104,7 @@ public class IncrementalIndexSchema
   {
     private long minTimestamp;
     private TimestampSpec timestampSpec;
-    private Granularity gran;
+    private Granularity queryGranularity;
     private VirtualColumns virtualColumns;
     private DimensionsSpec dimensionsSpec;
     private AggregatorFactory[] metrics;
@@ -113,7 +113,7 @@ public class IncrementalIndexSchema
     public Builder()
     {
       this.minTimestamp = 0L;
-      this.gran = Granularities.NONE;
+      this.queryGranularity = Granularities.NONE;
       this.virtualColumns = VirtualColumns.EMPTY;
       this.dimensionsSpec = DimensionsSpec.EMPTY;
       this.metrics = new AggregatorFactory[]{};
@@ -134,7 +134,7 @@ public class IncrementalIndexSchema
 
     public Builder withQueryGranularity(Granularity gran)
     {
-      this.gran = gran;
+      this.queryGranularity = gran;
       return this;
     }
 
@@ -181,7 +181,7 @@ public class IncrementalIndexSchema
       return new IncrementalIndexSchema(
           minTimestamp,
           timestampSpec,
-          gran,
+          queryGranularity,
           virtualColumns,
           dimensionsSpec,
           metrics,
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
index 8d72133a205..ad55e33adae 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
@@ -19,36 +19,23 @@
 
 package org.apache.druid.segment.incremental;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
-import org.apache.druid.query.BaseQuery;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
-import org.apache.druid.query.filter.ValueMatcher;
-import org.apache.druid.segment.ColumnSelectorFactory;
-import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.DimensionIndexer;
 import org.apache.druid.segment.Metadata;
 import org.apache.druid.segment.NestedDataColumnIndexerV4;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.ListIndexed;
-import org.apache.druid.segment.filter.ValueMatchers;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
-import java.util.Iterator;
 
 /**
  */
@@ -256,37 +243,9 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
   }
 
   @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable final Filter filter,
-      final Interval interval,
-      final VirtualColumns virtualColumns,
-      final Granularity gran,
-      final boolean descending,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    if (index.isEmpty()) {
-      return Sequences.empty();
-    }
-
-    if (queryMetrics != null) {
-      queryMetrics.vectorized(false);
-    }
-
-    final Interval dataInterval = new Interval(getMinTime(), gran.bucketEnd(getMaxTime()));
-
-    if (!interval.overlaps(dataInterval)) {
-      return Sequences.empty();
-    }
-    final Interval actualInterval = interval.overlap(dataInterval);
-    Iterable<Interval> intervals = gran.getIterable(actualInterval);
-    if (descending) {
-      intervals = Lists.reverse(ImmutableList.copyOf(intervals));
-    }
-
-    return Sequences
-        .simple(intervals)
-        .map(i -> new IncrementalIndexCursor(virtualColumns, descending, filter, i, actualInterval, gran));
+    return new IncrementalIndexCursorHolder(this, index, spec);
   }
 
   @Override
@@ -294,168 +253,4 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
   {
     return index.getMetadata();
   }
-
-  private class IncrementalIndexCursor implements Cursor
-  {
-    private IncrementalIndexRowHolder currEntry;
-    private final ColumnSelectorFactory columnSelectorFactory;
-    private final ValueMatcher filterMatcher;
-    private final int maxRowIndex;
-    private Iterator<IncrementalIndexRow> baseIter;
-    private Iterable<IncrementalIndexRow> cursorIterable;
-    private boolean emptyRange;
-    private final DateTime time;
-    private int numAdvanced;
-    private boolean done;
-
-    IncrementalIndexCursor(
-        VirtualColumns virtualColumns,
-        boolean descending,
-        Filter filter,
-        Interval interval,
-        Interval actualInterval,
-        Granularity gran
-    )
-    {
-      currEntry = new IncrementalIndexRowHolder();
-      columnSelectorFactory = new IncrementalIndexColumnSelectorFactory(
-          IncrementalIndexStorageAdapter.this,
-          virtualColumns,
-          descending,
-          currEntry
-      );
-      // Set maxRowIndex before creating the filterMatcher. See https://github.com/apache/druid/pull/6340
-      maxRowIndex = index.getLastRowIndex();
-      filterMatcher = filter == null ? ValueMatchers.allTrue() : filter.makeMatcher(columnSelectorFactory);
-      numAdvanced = -1;
-      final long timeStart = Math.max(interval.getStartMillis(), actualInterval.getStartMillis());
-      cursorIterable = index.getFacts().timeRangeIterable(
-          descending,
-          timeStart,
-          Math.min(actualInterval.getEndMillis(), gran.increment(interval.getStartMillis()))
-      );
-      emptyRange = !cursorIterable.iterator().hasNext();
-      time = gran.toDateTime(interval.getStartMillis());
-
-      reset();
-    }
-
-    @Override
-    public ColumnSelectorFactory getColumnSelectorFactory()
-    {
-      return columnSelectorFactory;
-    }
-
-    @Override
-    public DateTime getTime()
-    {
-      return time;
-    }
-
-    @Override
-    public void advance()
-    {
-      if (!baseIter.hasNext()) {
-        done = true;
-        return;
-      }
-
-      while (baseIter.hasNext()) {
-        BaseQuery.checkInterrupted();
-
-        IncrementalIndexRow entry = baseIter.next();
-        if (beyondMaxRowIndex(entry.getRowIndex())) {
-          continue;
-        }
-
-        currEntry.set(entry);
-
-        if (filterMatcher.matches(false)) {
-          return;
-        }
-      }
-
-      done = true;
-    }
-
-    @Override
-    public void advanceUninterruptibly()
-    {
-      if (!baseIter.hasNext()) {
-        done = true;
-        return;
-      }
-
-      while (baseIter.hasNext()) {
-        if (Thread.currentThread().isInterrupted()) {
-          return;
-        }
-
-        IncrementalIndexRow entry = baseIter.next();
-        if (beyondMaxRowIndex(entry.getRowIndex())) {
-          continue;
-        }
-
-        currEntry.set(entry);
-
-        if (filterMatcher.matches(false)) {
-          return;
-        }
-      }
-
-      done = true;
-    }
-
-    @Override
-    public boolean isDone()
-    {
-      return done;
-    }
-
-    @Override
-    public boolean isDoneOrInterrupted()
-    {
-      return isDone() || Thread.currentThread().isInterrupted();
-    }
-
-    @Override
-    public void reset()
-    {
-      baseIter = cursorIterable.iterator();
-
-      if (numAdvanced == -1) {
-        numAdvanced = 0;
-      } else {
-        Iterators.advance(baseIter, numAdvanced);
-      }
-
-      BaseQuery.checkInterrupted();
-
-      boolean foundMatched = false;
-      while (baseIter.hasNext()) {
-        IncrementalIndexRow entry = baseIter.next();
-        if (beyondMaxRowIndex(entry.getRowIndex())) {
-          numAdvanced++;
-          continue;
-        }
-        currEntry.set(entry);
-        if (filterMatcher.matches(false)) {
-          foundMatched = true;
-          break;
-        }
-
-        numAdvanced++;
-      }
-
-      done = !foundMatched && (emptyRange || !baseIter.hasNext());
-    }
-
-    private boolean beyondMaxRowIndex(int rowIndex)
-    {
-      // ignore rows whose rowIndex is beyond the maxRowIndex
-      // rows are order by timestamp, not rowIndex,
-      // so we still need to go through all rows to skip rows added after cursor created
-      return rowIndex > maxRowIndex;
-    }
-  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/join/HashJoinEngine.java b/processing/src/main/java/org/apache/druid/segment/join/HashJoinEngine.java
index 9efbcaecdee..6567baa140d 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/HashJoinEngine.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/HashJoinEngine.java
@@ -28,7 +28,6 @@ import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.RowIdSupplier;
 import org.apache.druid.segment.column.ColumnCapabilities;
-import org.joda.time.DateTime;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -167,13 +166,6 @@ public class HashJoinEngine
         return joinColumnSelectorFactory;
       }
 
-      @Override
-      @Nonnull
-      public DateTime getTime()
-      {
-        return leftCursor.getTime();
-      }
-
       @Override
       public void advance()
       {
@@ -256,6 +248,7 @@ public class HashJoinEngine
         leftCursor.reset();
         joinMatcher.reset();
         joinColumnSelectorFactory.resetRowId();
+        initialize();
       }
     }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
index 9dade664f8a..1276a60523f 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
@@ -23,14 +23,13 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
-import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.query.QueryMetrics;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.Metadata;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
@@ -42,11 +41,10 @@ import org.apache.druid.segment.join.filter.JoinFilterAnalyzer;
 import org.apache.druid.segment.join.filter.JoinFilterPreAnalysis;
 import org.apache.druid.segment.join.filter.JoinFilterPreAnalysisKey;
 import org.apache.druid.segment.join.filter.JoinFilterSplit;
-import org.apache.druid.segment.vector.VectorCursor;
+import org.apache.druid.utils.CloseableUtils;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
-import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.Arrays;
 import java.util.LinkedHashSet;
@@ -219,12 +217,117 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
   }
 
   @Override
-  public boolean canVectorize(@Nullable Filter filter, VirtualColumns virtualColumns, boolean descending)
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    // HashJoinEngine isn't vectorized yet.
-    // However, we can still vectorize if there are no clauses, since that means all we need to do is apply
-    // a base filter. That's easy enough!
-    return clauses.isEmpty() && baseAdapter.canVectorize(baseFilterAnd(filter), virtualColumns, descending);
+    // make a copy of CursorBuildSpec with filters removed
+    final CursorBuildSpec.CursorBuildSpecBuilder cursorBuildSpecBuilder = CursorBuildSpec.builder(spec)
+                                                                                         .setFilter(null);
+
+    final Filter combinedFilter = baseFilterAnd(spec.getFilter());
+
+    if (clauses.isEmpty()) {
+      // if there are no clauses, we can just use the base cursor directly if we apply the combined filter
+      final CursorBuildSpec newSpec = cursorBuildSpecBuilder.setFilter(combinedFilter)
+                                                            .build();
+      return baseAdapter.makeCursorHolder(newSpec);
+    }
+
+    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    final List<OrderBy> ordering;
+    final boolean descending;
+    if (Cursors.preferDescendingTimeOrdering(spec)) {
+      ordering = Cursors.descendingTimeOrder();
+      descending = true;
+    } else {
+      ordering = Cursors.ascendingTimeOrder();
+      descending = false;
+    }
+
+    return new CursorHolder()
+    {
+      final Closer joinablesCloser = Closer.create();
+
+      @Override
+      public Cursor asCursor()
+      {
+        // Filter pre-analysis key implied by the call to "makeCursorHolder". We need to sanity-check that it matches
+        // the actual pre-analysis that was done. Note: we could now infer a rewrite config from the "makeCursorHolder"
+        // call (it requires access to the query context which we now have access to since the move away from
+        // CursorFactory) but this code hasn't been updated to sanity-check it, so currently we are still skipping it
+        // by re-using the one present in the cached key.
+        final JoinFilterPreAnalysisKey keyIn =
+            new JoinFilterPreAnalysisKey(
+                joinFilterPreAnalysis.getKey().getRewriteConfig(),
+                clauses,
+                spec.getVirtualColumns(),
+                combinedFilter
+            );
+
+        final JoinFilterPreAnalysisKey keyCached = joinFilterPreAnalysis.getKey();
+        final JoinFilterPreAnalysis preAnalysis;
+        if (keyIn.equals(keyCached)) {
+          // Common case: key used during filter pre-analysis (keyCached) matches key implied by makeCursorHolder call
+          // (keyIn).
+          preAnalysis = joinFilterPreAnalysis;
+        } else {
+          // Less common case: key differs. Re-analyze the filter. This case can happen when an unnest datasource is
+          // layered on top of a join datasource.
+          preAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(keyIn);
+        }
+
+        final JoinFilterSplit joinFilterSplit = JoinFilterAnalyzer.splitFilter(
+            preAnalysis,
+            baseFilter
+        );
+
+
+        if (joinFilterSplit.getBaseTableFilter().isPresent()) {
+          cursorBuildSpecBuilder.setFilter(joinFilterSplit.getBaseTableFilter().get());
+        }
+        final VirtualColumns preJoinVirtualColumns = VirtualColumns.fromIterable(
+            Iterables.concat(
+                Sets.difference(
+                    ImmutableSet.copyOf(spec.getVirtualColumns().getVirtualColumns()),
+                    joinFilterPreAnalysis.getPostJoinVirtualColumns()
+                ),
+                joinFilterSplit.getPushDownVirtualColumns()
+            )
+        );
+        cursorBuildSpecBuilder.setVirtualColumns(preJoinVirtualColumns);
+
+        final Cursor baseCursor = joinablesCloser.register(baseAdapter.makeCursorHolder(cursorBuildSpecBuilder.build()))
+                                                 .asCursor();
+
+        if (baseCursor == null) {
+          return null;
+        }
+
+        Cursor retVal = baseCursor;
+
+        for (JoinableClause clause : clauses) {
+          retVal = HashJoinEngine.makeJoinCursor(retVal, clause, descending, joinablesCloser);
+        }
+
+        return PostJoinCursor.wrap(
+            retVal,
+            VirtualColumns.fromIterable(preAnalysis.getPostJoinVirtualColumns()),
+            joinFilterSplit.getJoinTableFilter().orElse(null)
+        );
+      }
+
+      @Nullable
+      @Override
+      public List<OrderBy> getOrdering()
+      {
+        return ordering;
+      }
+
+      @Override
+      public void close()
+      {
+        CloseableUtils.closeAndWrapExceptions(joinablesCloser);
+      }
+    };
   }
 
   @Override
@@ -233,123 +336,6 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
     return baseAdapter.isFromTombstone();
   }
 
-  @Nullable
-  @Override
-  public VectorCursor makeVectorCursor(
-      @Nullable Filter filter,
-      Interval interval,
-      VirtualColumns virtualColumns,
-      boolean descending,
-      int vectorSize,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
-  {
-    if (!canVectorize(filter, virtualColumns, descending)) {
-      throw new ISE("Cannot vectorize. Check 'canVectorize' before calling 'makeVectorCursor'.");
-    }
-
-    // Should have been checked by canVectorize.
-    assert clauses.isEmpty();
-
-    return baseAdapter.makeVectorCursor(
-        baseFilterAnd(filter),
-        interval,
-        virtualColumns,
-        descending,
-        vectorSize,
-        queryMetrics
-    );
-  }
-
-  @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable final Filter filter,
-      @Nonnull final Interval interval,
-      @Nonnull final VirtualColumns virtualColumns,
-      @Nonnull final Granularity gran,
-      final boolean descending,
-      @Nullable final QueryMetrics<?> queryMetrics
-  )
-  {
-    final Filter combinedFilter = baseFilterAnd(filter);
-
-    if (clauses.isEmpty()) {
-      return baseAdapter.makeCursors(
-          combinedFilter,
-          interval,
-          virtualColumns,
-          gran,
-          descending,
-          queryMetrics
-      );
-    }
-
-    // Filter pre-analysis key implied by the call to "makeCursors". We need to sanity-check that it matches
-    // the actual pre-analysis that was done. Note: we can't infer a rewrite config from the "makeCursors" call (it
-    // requires access to the query context) so we'll need to skip sanity-checking it, by re-using the one present
-    // in the cached key.)
-    final JoinFilterPreAnalysisKey keyIn =
-        new JoinFilterPreAnalysisKey(
-            joinFilterPreAnalysis.getKey().getRewriteConfig(),
-            clauses,
-            virtualColumns,
-            combinedFilter
-        );
-
-    final JoinFilterPreAnalysisKey keyCached = joinFilterPreAnalysis.getKey();
-    final JoinFilterPreAnalysis preAnalysis;
-    if (keyIn.equals(keyCached)) {
-      // Common case: key used during filter pre-analysis (keyCached) matches key implied by makeCursors call (keyIn).
-      preAnalysis = joinFilterPreAnalysis;
-    } else {
-      // Less common case: key differs. Re-analyze the filter. This case can happen when an unnest datasource is
-      // layered on top of a join datasource.
-      preAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(keyIn);
-    }
-
-
-    final JoinFilterSplit joinFilterSplit = JoinFilterAnalyzer.splitFilter(
-        preAnalysis,
-        baseFilter
-    );
-
-    final Sequence<Cursor> baseCursorSequence = baseAdapter.makeCursors(
-        joinFilterSplit.getBaseTableFilter().isPresent() ? joinFilterSplit.getBaseTableFilter().get() : null,
-        interval,
-        VirtualColumns.fromIterable(
-            Iterables.concat(
-                Sets.difference(
-                    ImmutableSet.copyOf(virtualColumns.getVirtualColumns()),
-                    joinFilterPreAnalysis.getPostJoinVirtualColumns()
-                ),
-                joinFilterSplit.getPushDownVirtualColumns()
-            )
-        ),
-        gran,
-        descending,
-        queryMetrics
-    );
-
-    final Closer joinablesCloser = Closer.create();
-    return Sequences.<Cursor, Cursor>map(
-        baseCursorSequence,
-        cursor -> {
-          assert cursor != null;
-          Cursor retVal = cursor;
-
-          for (JoinableClause clause : clauses) {
-            retVal = HashJoinEngine.makeJoinCursor(retVal, clause, descending, joinablesCloser);
-          }
-
-          return PostJoinCursor.wrap(
-              retVal,
-              VirtualColumns.fromIterable(preAnalysis.getPostJoinVirtualColumns()),
-              joinFilterSplit.getJoinTableFilter().orElse(null)
-          );
-        }
-    ).withBaggage(joinablesCloser);
-  }
-
   /**
    * Returns whether "column" will be selected from "baseAdapter". This is true if it is not shadowed by any joinables
    * (i.e. if it does not start with any of their prefixes).
@@ -368,7 +354,7 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
    */
   private Optional<JoinableClause> getClauseForColumn(final String column)
   {
-    // Check clauses in reverse, since "makeCursors" creates the cursor in such a way that the last clause
+    // Check clauses in reverse, since "makeCursorHolder" creates the cursor in such a way that the last clause
     // gets first dibs to claim a column.
     return Lists.reverse(clauses)
                 .stream()
diff --git a/processing/src/main/java/org/apache/druid/segment/join/PostJoinCursor.java b/processing/src/main/java/org/apache/druid/segment/join/PostJoinCursor.java
index 26c119bd34f..b8ef88996ff 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/PostJoinCursor.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/PostJoinCursor.java
@@ -24,14 +24,14 @@ import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.VirtualColumns;
-import org.joda.time.DateTime;
 
 import javax.annotation.Nullable;
 
 /**
- * A Cursor decorator used by {@link HashJoinSegmentStorageAdapter#makeCursors} to add post-join virtual columns
- * and filters.
+ * A Cursor decorator used by {@link HashJoinSegmentStorageAdapter#makeCursorHolder(CursorBuildSpec)} to add post-join
+ * virtual columns and filters.
  */
 public class PostJoinCursor implements Cursor
 {
@@ -105,12 +105,6 @@ public class PostJoinCursor implements Cursor
     return columnSelectorFactory;
   }
 
-  @Override
-  public DateTime getTime()
-  {
-    return baseCursor.getTime();
-  }
-
   @Nullable
   public Filter getPostJoinFilter()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java b/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
index de306209f90..716decb04d9 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
@@ -21,9 +21,6 @@ package org.apache.druid.segment.join.table;
 
 import com.google.common.base.Preconditions;
 import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.cache.CacheKeyBuilder;
@@ -31,6 +28,8 @@ import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnCache;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.NilColumnValueSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexColumnSelectorFactory;
@@ -44,12 +43,12 @@ import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.ReadableOffset;
 import org.apache.druid.timeline.SegmentId;
-import org.joda.time.chrono.ISOChronology;
 
 import javax.annotation.Nullable;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -113,67 +112,54 @@ public class BroadcastSegmentIndexedTable implements IndexedTable
       indexBuilders.add(m);
     }
 
-    // sort of like the dump segment tool, but build key column indexes when reading the segment
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        queryableIndex.getDataInterval().withChronology(ISOChronology.getInstanceUTC()),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      if (cursor == null) {
+        this.keyColumnsIndexes = Collections.emptyList();
+        return;
+      }
 
-    final Sequence<Integer> sequence = Sequences.map(
-        cursors,
-        cursor -> {
-          if (cursor == null) {
-            return 0;
-          }
-          int rowNumber = 0;
-          ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+      int rowNumber = 0;
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-          // this should really be optimized to use dimension selectors where possible to populate indexes from bitmap
-          // indexes, but, an optimization for another day
-          final List<BaseObjectColumnValueSelector> selectors = keyColumnNames
-              .stream()
-              .map(columnName -> {
-                // multi-value dimensions are not currently supported
-                if (adapter.getColumnCapabilities(columnName).hasMultipleValues().isMaybeTrue()) {
-                  return NilColumnValueSelector.instance();
-                }
-                return columnSelectorFactory.makeColumnValueSelector(columnName);
-              })
-              .collect(Collectors.toList());
-
-          while (!cursor.isDone()) {
-            for (int keyColumnSelectorIndex = 0; keyColumnSelectorIndex < selectors.size(); keyColumnSelectorIndex++) {
-              final String keyColumnName = keyColumnNames.get(keyColumnSelectorIndex);
-              final int columnPosition = rowSignature.indexOf(keyColumnName);
-              final RowBasedIndexBuilder keyColumnIndexBuilder = indexBuilders.get(columnPosition);
-              keyColumnIndexBuilder.add(selectors.get(keyColumnSelectorIndex).getObject());
+      // this should really be optimized to use dimension selectors where possible to populate indexes from bitmap
+      // indexes, but, an optimization for another day
+      final List<BaseObjectColumnValueSelector> selectors = keyColumnNames
+          .stream()
+          .map(columnName -> {
+            // multi-value dimensions are not currently supported
+            if (adapter.getColumnCapabilities(columnName).hasMultipleValues().isMaybeTrue()) {
+              return NilColumnValueSelector.instance();
             }
+            return columnSelectorFactory.makeColumnValueSelector(columnName);
+          })
+          .collect(Collectors.toList());
 
-            if (rowNumber % 100_000 == 0) {
-              if (rowNumber == 0) {
-                LOG.debug("Indexed first row for table %s", theSegment.getId());
-              } else {
-                LOG.debug("Indexed row %s for table %s", rowNumber, theSegment.getId());
-              }
-            }
-            rowNumber++;
-            cursor.advance();
-          }
-          return rowNumber;
+      while (!cursor.isDone()) {
+        for (int keyColumnSelectorIndex = 0; keyColumnSelectorIndex < selectors.size(); keyColumnSelectorIndex++) {
+          final String keyColumnName = keyColumnNames.get(keyColumnSelectorIndex);
+          final int columnPosition = rowSignature.indexOf(keyColumnName);
+          final RowBasedIndexBuilder keyColumnIndexBuilder = indexBuilders.get(columnPosition);
+          keyColumnIndexBuilder.add(selectors.get(keyColumnSelectorIndex).getObject());
         }
-    );
 
-    Integer totalRows = sequence.accumulate(0, (accumulated, in) -> accumulated += in);
+        if (rowNumber % 100_000 == 0) {
+          if (rowNumber == 0) {
+            LOG.debug("Indexed first row for table %s", theSegment.getId());
+          } else {
+            LOG.debug("Indexed row %s for table %s", rowNumber, theSegment.getId());
+          }
+        }
+        rowNumber++;
+        cursor.advance();
+      }
 
-    this.keyColumnsIndexes = indexBuilders.stream()
-                                          .map(builder -> builder != null ? builder.build() : null)
-                                          .collect(Collectors.toList());
+      this.keyColumnsIndexes = indexBuilders.stream()
+                                            .map(builder -> builder != null ? builder.build() : null)
+                                            .collect(Collectors.toList());
 
-    LOG.info("Created BroadcastSegmentIndexedTable with %s rows.", totalRows);
+      LOG.info("Created BroadcastSegmentIndexedTable with %s rows.", rowNumber);
+    }
   }
 
   @Override
diff --git a/processing/src/main/java/org/apache/druid/segment/join/table/FrameBasedIndexedTable.java b/processing/src/main/java/org/apache/druid/segment/join/table/FrameBasedIndexedTable.java
index 3e2b46d5f68..080a3c920ae 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/table/FrameBasedIndexedTable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/table/FrameBasedIndexedTable.java
@@ -30,7 +30,6 @@ import org.apache.druid.frame.segment.columnar.FrameQueryableIndex;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.logger.Logger;
@@ -39,10 +38,11 @@ import org.apache.druid.query.FrameSignaturePair;
 import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.NilColumnValueSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.SimpleAscendingOffset;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.BaseColumn;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
@@ -114,62 +114,58 @@ public class FrameBasedIndexedTable implements IndexedTable
       indexBuilders.add(m);
     }
 
-    final Sequence<Cursor> cursors = Sequences.concat(
+    final Sequence<FrameStorageAdapter> storageAdapters = Sequences.simple(
         frameBasedInlineDataSource
             .getFrames()
             .stream()
             .map(frameSignaturePair -> {
               Frame frame = frameSignaturePair.getFrame();
               RowSignature rowSignature = frameSignaturePair.getRowSignature();
-              FrameStorageAdapter frameStorageAdapter =
-                  new FrameStorageAdapter(frame, FrameReader.create(rowSignature), Intervals.ETERNITY);
-              return frameStorageAdapter.makeCursors(
-                                            null,
-                                            Intervals.ETERNITY,
-                                            VirtualColumns.EMPTY,
-                                            Granularities.ALL,
-                                            false,
-                                            null
-                                        );
+              return new FrameStorageAdapter(frame, FrameReader.create(rowSignature), Intervals.ETERNITY);
             })
             .collect(Collectors.toList())
     );
 
     final Sequence<Integer> sequence = Sequences.map(
-        cursors,
-        cursor -> {
-          if (cursor == null) {
-            return 0;
-          }
-          int rowNumber = 0;
-          ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
-
-          // this should really be optimized to use dimension selectors where possible to populate indexes from bitmap
-          // indexes, but, an optimization for another day
-          final List<BaseObjectColumnValueSelector> selectors = keyColumnNames
-              .stream()
-              .map(columnSelectorFactory::makeColumnValueSelector)
-              .collect(Collectors.toList());
-
-          while (!cursor.isDone()) {
-            for (int keyColumnSelectorIndex = 0; keyColumnSelectorIndex < selectors.size(); keyColumnSelectorIndex++) {
-              final String keyColumnName = keyColumnNames.get(keyColumnSelectorIndex);
-              final int columnPosition = rowSignature.indexOf(keyColumnName);
-              final RowBasedIndexBuilder keyColumnIndexBuilder = indexBuilders.get(columnPosition);
-              keyColumnIndexBuilder.add(selectors.get(keyColumnSelectorIndex).getObject());
+        storageAdapters,
+        storageAdapter -> {
+          try (final CursorHolder holder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+            final Cursor cursor = holder.asCursor();
+            if (cursor == null) {
+              return 0;
             }
 
-            if (rowNumber % 100_000 == 0) {
-              if (rowNumber == 0) {
-                LOG.debug("Indexed first row for frame based datasource");
-              } else {
-                LOG.debug("Indexed row %s for frame based datasource", rowNumber);
+            int rowNumber = 0;
+            ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+
+            // this should really be optimized to use dimension selectors where possible to populate indexes from bitmap
+            // indexes, but, an optimization for another day
+            final List<BaseObjectColumnValueSelector> selectors = keyColumnNames
+                .stream()
+                .map(columnSelectorFactory::makeColumnValueSelector)
+                .collect(Collectors.toList());
+
+            while (!cursor.isDone()) {
+              for (int keyColumnSelectorIndex = 0; keyColumnSelectorIndex
+                                                   < selectors.size(); keyColumnSelectorIndex++) {
+                final String keyColumnName = keyColumnNames.get(keyColumnSelectorIndex);
+                final int columnPosition = rowSignature.indexOf(keyColumnName);
+                final RowBasedIndexBuilder keyColumnIndexBuilder = indexBuilders.get(columnPosition);
+                keyColumnIndexBuilder.add(selectors.get(keyColumnSelectorIndex).getObject());
               }
+
+              if (rowNumber % 100_000 == 0) {
+                if (rowNumber == 0) {
+                  LOG.debug("Indexed first row for frame based datasource");
+                } else {
+                  LOG.debug("Indexed row %s for frame based datasource", rowNumber);
+                }
+              }
+              rowNumber++;
+              cursor.advance();
             }
-            rowNumber++;
-            cursor.advance();
+            return rowNumber;
           }
-          return rowNumber;
         }
     );
 
diff --git a/processing/src/main/java/org/apache/druid/segment/vector/VectorCursor.java b/processing/src/main/java/org/apache/druid/segment/vector/VectorCursor.java
index fde58855a53..103d769fc47 100644
--- a/processing/src/main/java/org/apache/druid/segment/vector/VectorCursor.java
+++ b/processing/src/main/java/org/apache/druid/segment/vector/VectorCursor.java
@@ -19,22 +19,22 @@
 
 package org.apache.druid.segment.vector;
 
-import java.io.Closeable;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolderFactory;
 
 /**
- * Vectorized cursor used during query execution. VectorCursors are returned by
- * {@link org.apache.druid.segment.StorageAdapter#makeVectorCursor} and are created by
- * {@link org.apache.druid.segment.QueryableIndexCursorSequenceBuilder#buildVectorized}.
- *
- * Unlike the non-vectorized version, VectorCursor does not have a getTime() method. This is because we are trying to
- * avoid creating needlessly-small vectors when the time granularity is very fine. See
- * {@link org.apache.druid.query.vector.VectorCursorGranularizer} for a helper that makes it easier for query engines to
- * do their own time granularization.
- *
+ * Vectorized cursor used during query execution. VectorCursors are available from
+ * {@link CursorHolderFactory#makeCursorHolder(CursorBuildSpec)} via
+ * {@link org.apache.druid.segment.CursorHolder#asVectorCursor()}.
+ * <p>
+ * See {@link org.apache.druid.query.vector.VectorCursorGranularizer} for a helper that makes it easier for query
+ * engines to do time granularization.
+ * <p>
  * An example of how to use the methods in this class:
  *
  * <pre>
- *   try (VectorCursor cursor = adapter.makeVectorCursor(...)) {
+ *   try (CursorHolder cursorHolder = adapter.makeCursorHolder(...)) {
+ *     VectorCursor cursor = cursorHolder.asVectorCursor();
  *     // ProcessorClass is some vectorized processor class.
  *     ProcessorClass o = makeProcessor(cursor.getColumnSelectorFactory());
  *     for (; !cursor.isDone(); cursor.advance()) {
@@ -45,7 +45,7 @@ import java.io.Closeable;
  *
  * @see org.apache.druid.segment.Cursor, the non-vectorized version.
  */
-public interface VectorCursor extends VectorSizeInspector, Closeable
+public interface VectorCursor extends VectorSizeInspector
 {
   /**
    * Returns a vectorized column selector factory.
@@ -67,10 +67,4 @@ public interface VectorCursor extends VectorSizeInspector, Closeable
    */
   @SuppressWarnings("unused") /* Not currently used, but anticipated to be used by topN in the future. */
   void reset();
-
-  /**
-   * Close the cursor and release its resources.
-   */
-  @Override
-  void close();
 }
diff --git a/processing/src/main/java/org/apache/druid/utils/CollectionUtils.java b/processing/src/main/java/org/apache/druid/utils/CollectionUtils.java
index 15c11ebea55..bd7600f6c03 100644
--- a/processing/src/main/java/org/apache/druid/utils/CollectionUtils.java
+++ b/processing/src/main/java/org/apache/druid/utils/CollectionUtils.java
@@ -177,7 +177,7 @@ public final class CollectionUtils
   }
 
   /**
-   * Intersection of two sets: {@code C = A ∪ B}.
+   * Union of two sets: {@code C = A ∪ B}.
    */
   public static <T> Set<T> union(Set<T> left, Set<T> right)
   {
diff --git a/processing/src/test/java/org/apache/druid/frame/TestArrayStorageAdapter.java b/processing/src/test/java/org/apache/druid/frame/TestArrayStorageAdapter.java
index a8190e29f3a..173750a74c7 100644
--- a/processing/src/test/java/org/apache/druid/frame/TestArrayStorageAdapter.java
+++ b/processing/src/test/java/org/apache/druid/frame/TestArrayStorageAdapter.java
@@ -20,31 +20,28 @@
 package org.apache.druid.frame;
 
 import com.google.common.collect.Iterables;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.query.QueryMetrics;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.DimensionSpec;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.ObjectColumnSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.data.IndexedInts;
-import org.joda.time.DateTime;
-import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
+import java.util.List;
 
 /**
  * Storage adapter around {@link QueryableIndex} that transforms all multi-value strings columns into string arrays.
@@ -57,28 +54,33 @@ public class TestArrayStorageAdapter extends QueryableIndexStorageAdapter
   }
 
   @Override
-  public boolean canVectorize(
-      @Nullable Filter filter,
-      VirtualColumns virtualColumns,
-      boolean descending
-  )
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    return false;
+    final CursorHolder delegate = super.makeCursorHolder(spec);
+    return new CursorHolder()
+    {
+      @Nullable
+      @Override
+      public Cursor asCursor()
+      {
+        return new DecoratedCursor(delegate.asCursor());
+      }
+
+      @Nullable
+      @Override
+      public List<OrderBy> getOrdering()
+      {
+        return delegate.getOrdering();
+      }
+
+      @Override
+      public void close()
+      {
+        delegate.close();
+      }
+    };
   }
 
-  @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable final Filter filter,
-      final Interval interval,
-      final VirtualColumns virtualColumns,
-      final Granularity gran,
-      final boolean descending,
-      @Nullable final QueryMetrics<?> queryMetrics
-  )
-  {
-    return super.makeCursors(filter, interval, virtualColumns, gran, descending, queryMetrics)
-                .map(DecoratedCursor::new);
-  }
 
   @Override
   public RowSignature getRowSignature()
@@ -191,12 +193,6 @@ public class TestArrayStorageAdapter extends QueryableIndexStorageAdapter
       };
     }
 
-    @Override
-    public DateTime getTime()
-    {
-      return cursor.getTime();
-    }
-
     @Override
     public void advance()
     {
diff --git a/processing/src/test/java/org/apache/druid/frame/processor/OutputChannelFactoryTest.java b/processing/src/test/java/org/apache/druid/frame/processor/OutputChannelFactoryTest.java
index 16d155f3df3..1e65ed7142a 100644
--- a/processing/src/test/java/org/apache/druid/frame/processor/OutputChannelFactoryTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/processor/OutputChannelFactoryTest.java
@@ -31,10 +31,10 @@ import org.apache.druid.frame.segment.FrameStorageAdapter;
 import org.apache.druid.frame.testutil.FrameSequenceBuilder;
 import org.apache.druid.frame.testutil.FrameTestUtil;
 import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.TestIndex;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.Assert;
@@ -128,32 +128,31 @@ public abstract class OutputChannelFactoryTest extends InitializedNullHandlingTe
     Assert.assertTrue(readableFrameChannel.isFinished());
     readableFrameChannel.close();
 
-    // build list of rows from written and read data to verify
-    List<List<Object>> writtenData = adapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    ).flatMap(cursor -> FrameTestUtil.readRowsFromCursor(cursor, adapter.getRowSignature())).toList();
-
     FrameStorageAdapter frameStorageAdapter = new FrameStorageAdapter(
         readbackFrame,
         FrameReader.create(adapter.getRowSignature()),
         Intervals.ETERNITY
     );
-    List<List<Object>> readData = frameStorageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    ).flatMap(cursor -> FrameTestUtil.readRowsFromCursor(cursor, adapter.getRowSignature())).toList();
+    // build list of rows from written and read data to verify
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN);
+         final CursorHolder frameMaker = frameStorageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)
+    ) {
+      List<List<Object>> writtenData = FrameTestUtil.readRowsFromCursor(
+          cursorHolder.asCursor(),
+          adapter.getRowSignature()
+      ).toList();
+      List<List<Object>> readData = FrameTestUtil.readRowsFromCursor(
+          frameMaker.asCursor(),
+          frameStorageAdapter.getRowSignature()
+      ).toList();
 
-    Assert.assertEquals("Read rows count is different from written rows count", writtenData.size(), readData.size());
-    Assert.assertEquals("Read data is different from written data", writtenData, readData);
+      Assert.assertEquals(
+          "Read rows count is different from written rows count",
+          writtenData.size(),
+          readData.size()
+      );
+      Assert.assertEquals("Read data is different from written data", writtenData, readData);
+    }
   }
 
   @Test
diff --git a/processing/src/test/java/org/apache/druid/frame/segment/FrameStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/frame/segment/FrameStorageAdapterTest.java
index 481851a0f75..b9abf2f81b1 100644
--- a/processing/src/test/java/org/apache/druid/frame/segment/FrameStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/segment/FrameStorageAdapterTest.java
@@ -20,20 +20,25 @@
 package org.apache.druid.frame.segment;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import org.apache.druid.frame.FrameType;
 import org.apache.druid.frame.testutil.FrameTestUtil;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.extraction.TimeFormatExtractionFn;
 import org.apache.druid.query.extraction.UpperExtractionFn;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.StorageAdapter;
@@ -51,7 +56,6 @@ import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.joda.time.Interval;
 import org.junit.After;
 import org.junit.Assert;
-import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -67,7 +71,7 @@ import java.util.function.Function;
 public class FrameStorageAdapterTest
 {
   /**
-   * Basic tests: everything except makeCursors, makeVectorCursor.
+   * Basic tests: everything except makeCursor, makeVectorCursor.
    */
   @RunWith(Parameterized.class)
   public static class BasicTests extends InitializedNullHandlingTest
@@ -227,7 +231,7 @@ public class FrameStorageAdapterTest
   }
 
   /**
-   * CursorTests: matrix of tests of makeCursors, makeVectorCursor
+   * CursorTests: matrix of tests of makeCursor, makeVectorCursor
    */
   @RunWith(Parameterized.class)
   public static class CursorTests extends InitializedNullHandlingTest
@@ -244,6 +248,11 @@ public class FrameStorageAdapterTest
     private StorageAdapter queryableAdapter;
     private FrameSegment frameSegment;
     private StorageAdapter frameAdapter;
+    private final QueryContext queryContext = QueryContext.of(
+        ImmutableMap.of(QueryContexts.VECTOR_SIZE_KEY, VECTOR_SIZE)
+    );
+
+    private CursorBuildSpec buildSpec;
 
     public CursorTests(
         FrameType frameType,
@@ -258,6 +267,13 @@ public class FrameStorageAdapterTest
       this.interval = interval;
       this.virtualColumns = virtualColumns;
       this.descending = descending;
+      this.buildSpec = CursorBuildSpec.builder()
+                                      .setFilter(this.filter)
+                                      .setInterval(this.interval)
+                                      .setVirtualColumns(this.virtualColumns)
+                                      .setPreferredOrdering(descending ? Cursors.descendingTimeOrder() : null)
+                                      .setQueryContext(queryContext)
+                                      .build();
     }
 
     @Parameterized.Parameters(name = "frameType = {0}, "
@@ -345,58 +361,46 @@ public class FrameStorageAdapterTest
     }
 
     @Test
-    public void test_makeCursors()
+    public void test_makeCursor()
     {
-      assertCursorsMatch(
-          adapter ->
-              adapter.makeCursors(
-                  filter,
-                  interval,
-                  virtualColumns,
-                  Granularities.ALL, // Frames only support Granularities.ALL: no point testing the others.
-                  descending,
-                  null
-              )
-      );
+      assertCursorMatch(adapter -> adapter.makeCursorHolder(buildSpec));
     }
 
     @Test
     public void test_makeVectorCursor()
     {
-      Assume.assumeTrue(frameAdapter.canVectorize(filter, virtualColumns, descending));
-
-      assertVectorCursorsMatch(
-          adapter ->
-              adapter.makeVectorCursor(
-                  filter,
-                  interval,
-                  virtualColumns,
-                  descending,
-                  VECTOR_SIZE,
-                  null
-              )
-      );
+      assertVectorCursorsMatch(adapter -> adapter.makeCursorHolder(buildSpec));
     }
 
-    private void assertCursorsMatch(final Function<StorageAdapter, Sequence<Cursor>> call)
+    private void assertCursorMatch(final Function<StorageAdapter, CursorHolder> call)
     {
       final RowSignature signature = frameAdapter.getRowSignature();
-      final Sequence<List<Object>> queryableRows =
-          call.apply(queryableAdapter).flatMap(cursor -> FrameTestUtil.readRowsFromCursor(cursor, signature));
-      final Sequence<List<Object>> frameRows =
-          call.apply(frameAdapter)
-              .flatMap(cursor -> FrameTestUtil.readRowsFromCursor(advanceAndReset(cursor), signature));
-      FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
+      try (final CursorHolder queryableMaker = call.apply(queryableAdapter);
+           final CursorHolder frameMaker = call.apply(frameAdapter)) {
+        final Sequence<List<Object>> queryableRows =
+            FrameTestUtil.readRowsFromCursor(advanceAndReset(queryableMaker.asCursor()), signature);
+        final Sequence<List<Object>> frameRows =
+            FrameTestUtil.readRowsFromCursor(advanceAndReset(frameMaker.asCursor()), signature);
+        FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
+      }
     }
 
-    private void assertVectorCursorsMatch(final Function<StorageAdapter, VectorCursor> call)
+    private void assertVectorCursorsMatch(final Function<StorageAdapter, CursorHolder> call)
     {
-      final RowSignature signature = frameAdapter.getRowSignature();
-      final Sequence<List<Object>> queryableRows =
-          FrameTestUtil.readRowsFromVectorCursor(call.apply(queryableAdapter), signature);
-      final Sequence<List<Object>> frameRows =
-          FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(call.apply(frameAdapter)), signature);
-      FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
+      final CursorHolder cursorHolder = call.apply(queryableAdapter);
+      final CursorHolder frameCursorHolder = call.apply(frameAdapter);
+      if (frameCursorHolder.canVectorize()) {
+        final RowSignature signature = frameAdapter.getRowSignature();
+        final Sequence<List<Object>> queryableRows =
+            FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(cursorHolder.asVectorCursor()), signature).withBaggage(cursorHolder);
+        final Sequence<List<Object>> frameRows =
+            FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(frameCursorHolder.asVectorCursor()), signature)
+                         .withBaggage(frameCursorHolder);
+        FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
+      } else {
+        cursorHolder.close();
+        frameCursorHolder.close();
+      }
     }
 
     /**
diff --git a/processing/src/test/java/org/apache/druid/frame/testutil/FrameSequenceBuilder.java b/processing/src/test/java/org/apache/druid/frame/testutil/FrameSequenceBuilder.java
index 573e5c52123..eb816225196 100644
--- a/processing/src/test/java/org/apache/druid/frame/testutil/FrameSequenceBuilder.java
+++ b/processing/src/test/java/org/apache/druid/frame/testutil/FrameSequenceBuilder.java
@@ -34,6 +34,7 @@ import org.apache.druid.java.util.common.guava.BaseSequence;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
@@ -138,67 +139,65 @@ public class FrameSequenceBuilder
       throw DruidException.defensive("Unrecognized frame type");
     }
 
-    final Sequence<Cursor> cursors = FrameTestUtil.makeCursorsForAdapter(adapter, populateRowNumber);
+    final CursorHolder cursorHolder = FrameTestUtil.makeCursorForAdapter(adapter, populateRowNumber);
+    final Cursor cursor = cursorHolder.asCursor();
+    return new BaseSequence<>(
+        new BaseSequence.IteratorMaker<Frame, Iterator<Frame>>()
+        {
+          @Override
+          public Iterator<Frame> make()
+          {
+            final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    return cursors.flatMap(
-        cursor -> new BaseSequence<>(
-            new BaseSequence.IteratorMaker<Frame, Iterator<Frame>>()
+            return new Iterator<Frame>()
             {
               @Override
-              public Iterator<Frame> make()
+              public boolean hasNext()
               {
-                final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+                return !cursor.isDone();
+              }
 
-                return new Iterator<Frame>()
-                {
-                  @Override
-                  public boolean hasNext()
-                  {
-                    return !cursor.isDone();
-                  }
+              @Override
+              public Frame next()
+              {
+                if (cursor.isDone()) {
+                  throw new NoSuchElementException();
+                }
 
-                  @Override
-                  public Frame next()
-                  {
-                    if (cursor.isDone()) {
-                      throw new NoSuchElementException();
-                    }
-
-                    try (final FrameWriter writer = frameWriterFactory.newFrameWriter(columnSelectorFactory)) {
-                      while (!cursor.isDone()) {
-                        if (!writer.addSelection()) {
-                          if (writer.getNumRows() == 0) {
-                            throw new FrameRowTooLargeException(allocator.capacity());
-                          }
-
-                          return makeFrame(writer);
-                        }
-
-                        cursor.advance();
-
-                        if (writer.getNumRows() >= maxRowsPerFrame) {
-                          return makeFrame(writer);
-                        }
+                try (final FrameWriter writer = frameWriterFactory.newFrameWriter(columnSelectorFactory)) {
+                  while (!cursor.isDone()) {
+                    if (!writer.addSelection()) {
+                      if (writer.getNumRows() == 0) {
+                        throw new FrameRowTooLargeException(allocator.capacity());
                       }
 
                       return makeFrame(writer);
                     }
+
+                    cursor.advance();
+
+                    if (writer.getNumRows() >= maxRowsPerFrame) {
+                      return makeFrame(writer);
+                    }
                   }
 
-                  private Frame makeFrame(final FrameWriter writer)
-                  {
-                    return Frame.wrap(writer.toByteArray());
-                  }
-                };
+                  return makeFrame(writer);
+                }
               }
 
-              @Override
-              public void cleanup(Iterator<Frame> iterFromMake)
+              private Frame makeFrame(final FrameWriter writer)
               {
-                // Nothing to do.
+                return Frame.wrap(writer.toByteArray());
               }
-            }
-        )
-    );
+            };
+          }
+
+          @Override
+          public void cleanup(Iterator<Frame> iterFromMake)
+          {
+            // Nothing to do.
+          }
+        }
+    ).withBaggage(cursorHolder);
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java b/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java
index 7402e6ef592..e1981f3618b 100644
--- a/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java
+++ b/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java
@@ -33,14 +33,16 @@ import org.apache.druid.frame.segment.FrameSegment;
 import org.apache.druid.frame.segment.FrameStorageAdapter;
 import org.apache.druid.frame.util.SettableLongVirtualColumn;
 import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.segment.ColumnProcessors;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
@@ -218,7 +220,7 @@ public class FrameTestUtil
 
   /**
    * Reads a sequence of rows from a frame channel using a non-vectorized cursor from
-   * {@link FrameStorageAdapter#makeCursors}.
+   * {@link FrameStorageAdapter#makeCursorHolder(CursorBuildSpec)}.
    *
    * @param channel     the channel
    * @param frameReader reader for this channel
@@ -230,10 +232,15 @@ public class FrameTestUtil
   {
     return new FrameChannelSequence(channel)
         .flatMap(
-            frame ->
-                new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY)
-                    .makeCursors(null, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null)
-                    .flatMap(cursor -> readRowsFromCursor(cursor, frameReader.signature()))
+            frame -> {
+              final CursorHolder cursorHolder = new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY)
+                  .makeCursorHolder(CursorBuildSpec.FULL_SCAN);
+              final Cursor cursor = cursorHolder.asCursor();
+              if (cursor == null) {
+                return Sequences.withBaggage(Sequences.empty(), cursorHolder);
+              }
+              return readRowsFromCursor(cursor, frameReader.signature()).withBaggage(cursorHolder);
+            }
         );
   }
 
@@ -254,20 +261,23 @@ public class FrameTestUtil
   )
   {
     final RowSignature signatureToUse = signature == null ? adapter.getRowSignature() : signature;
-    return makeCursorsForAdapter(adapter, populateRowNumber).flatMap(
-        cursor -> readRowsFromCursor(cursor, signatureToUse)
-    );
+    final CursorHolder cursorHolder = makeCursorForAdapter(adapter, populateRowNumber);
+    final Cursor cursor = cursorHolder.asCursor();
+    if (cursor == null) {
+      return Sequences.withBaggage(Sequences.empty(), cursorHolder);
+    }
+    return readRowsFromCursor(cursor, signatureToUse).withBaggage(cursorHolder);
   }
 
   /**
-   * Creates a single-Cursor Sequence from a storage adapter.
+   * Creates a Cursor and  from a storage adapter.
    *
    * If {@param populateRowNumber} is set, the row number will be populated into {@link #ROW_NUMBER_COLUMN}.
    *
    * @param adapter           the adapter
    * @param populateRowNumber whether to populate {@link #ROW_NUMBER_COLUMN}
    */
-  public static Sequence<Cursor> makeCursorsForAdapter(
+  public static CursorHolder makeCursorForAdapter(
       final StorageAdapter adapter,
       final boolean populateRowNumber
   )
@@ -283,14 +293,40 @@ public class FrameTestUtil
       virtualColumns = VirtualColumns.EMPTY;
     }
 
-    return adapter.makeCursors(null, Intervals.ETERNITY, virtualColumns, Granularities.ALL, false, null)
-                  .map(cursor -> {
-                    if (populateRowNumber) {
-                      return new RowNumberUpdatingCursor(cursor, rowNumberVirtualColumn);
-                    } else {
-                      return cursor;
-                    }
-                  });
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(virtualColumns)
+                                                     .build();
+
+    final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec);
+    if (populateRowNumber) {
+      return new CursorHolder()
+      {
+        @Nullable
+        @Override
+        public Cursor asCursor()
+        {
+          final Cursor cursor = cursorHolder.asCursor();
+          if (cursor == null) {
+            return null;
+          }
+          return new RowNumberUpdatingCursor(cursor, rowNumberVirtualColumn);
+        }
+
+        @Override
+        public List<OrderBy> getOrdering()
+        {
+          return cursorHolder.getOrdering();
+        }
+
+        @Override
+        public void close()
+        {
+          cursorHolder.close();
+        }
+      };
+    } else {
+      return cursorHolder;
+    }
   }
 
   public static Sequence<List<Object>> readRowsFromCursor(final Cursor cursor, final RowSignature signature)
@@ -326,46 +362,41 @@ public class FrameTestUtil
 
   public static Sequence<List<Object>> readRowsFromVectorCursor(final VectorCursor cursor, final RowSignature signature)
   {
-    try {
-      final VectorColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+    final VectorColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-      final List<Supplier<Object[]>> columnReaders = new ArrayList<>();
+    final List<Supplier<Object[]>> columnReaders = new ArrayList<>();
 
-      for (int i = 0; i < signature.size(); i++) {
-        final String columnName = signature.getColumnName(i);
-        final Supplier<Object[]> columnReader = ColumnProcessors.makeVectorProcessor(
-            columnName,
-            RowReadingVectorColumnProcessorFactory.INSTANCE,
-            columnSelectorFactory
-        );
+    for (int i = 0; i < signature.size(); i++) {
+      final String columnName = signature.getColumnName(i);
+      final Supplier<Object[]> columnReader = ColumnProcessors.makeVectorProcessor(
+          columnName,
+          RowReadingVectorColumnProcessorFactory.INSTANCE,
+          columnSelectorFactory
+      );
 
-        columnReaders.add(columnReader);
-      }
+      columnReaders.add(columnReader);
+    }
 
-      final List<List<Object>> retVal = new ArrayList<>();
+    final List<List<Object>> retVal = new ArrayList<>();
 
-      while (!cursor.isDone()) {
-        final int vectorSize = cursor.getCurrentVectorSize();
-        final List<Object[]> columns = columnReaders.stream().map(Supplier::get).collect(Collectors.toList());
+    while (!cursor.isDone()) {
+      final int vectorSize = cursor.getCurrentVectorSize();
+      final List<Object[]> columns = columnReaders.stream().map(Supplier::get).collect(Collectors.toList());
 
-        for (int i = 0; i < vectorSize; i++) {
-          final List<Object> row = new ArrayList<>();
+      for (int i = 0; i < vectorSize; i++) {
+        final List<Object> row = new ArrayList<>();
 
-          for (final Object[] column : columns) {
-            row.add(column[i]);
-          }
-
-          retVal.add(row);
+        for (final Object[] column : columns) {
+          row.add(column[i]);
         }
 
-        cursor.advance();
+        retVal.add(row);
       }
 
-      return Sequences.simple(retVal);
-    }
-    finally {
-      cursor.close();
+      cursor.advance();
     }
+
+    return Sequences.simple(retVal);
   }
 
   private static Supplier<Object> dimensionSelectorReader(final DimensionSelector selector)
diff --git a/processing/src/test/java/org/apache/druid/frame/testutil/RowNumberUpdatingCursor.java b/processing/src/test/java/org/apache/druid/frame/testutil/RowNumberUpdatingCursor.java
index 3a47434891c..34cdab25a48 100644
--- a/processing/src/test/java/org/apache/druid/frame/testutil/RowNumberUpdatingCursor.java
+++ b/processing/src/test/java/org/apache/druid/frame/testutil/RowNumberUpdatingCursor.java
@@ -22,7 +22,6 @@ package org.apache.druid.frame.testutil;
 import org.apache.druid.frame.util.SettableLongVirtualColumn;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
-import org.joda.time.DateTime;
 
 /**
  * Used by {@link FrameTestUtil#readRowsFromAdapter} and {@link FrameTestUtil#readRowsFromCursor}.
@@ -44,12 +43,6 @@ public class RowNumberUpdatingCursor implements Cursor
     return baseCursor.getColumnSelectorFactory();
   }
 
-  @Override
-  public DateTime getTime()
-  {
-    return baseCursor.getTime();
-  }
-
   @Override
   public void advance()
   {
diff --git a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java
index c45ffa69735..9d7a9b72553 100644
--- a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java
@@ -42,18 +42,19 @@ import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.RE;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde;
 import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.RowBasedSegment;
 import org.apache.druid.segment.RowIdSupplier;
 import org.apache.druid.segment.Segment;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
 import org.apache.druid.segment.column.ColumnType;
@@ -576,48 +577,45 @@ public class FrameWriterTest extends InitializedNullHandlingTest
       inputSegment = new FrameSegment(inputFrame, FrameReader.create(signature), SegmentId.dummy("xxx"));
     }
 
-    return inputSegment.asStorageAdapter()
-                       .makeCursors(null, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null)
-                       .accumulate(
-                           null,
-                           (retVal, cursor) -> {
-                             int numRows = 0;
-                             final FrameWriterFactory frameWriterFactory;
-                             if (FrameType.ROW_BASED.equals(outputFrameType)) {
-                               frameWriterFactory = FrameWriters.makeRowBasedFrameWriterFactory(
-                                   new SingleMemoryAllocatorFactory(allocator),
-                                   signature,
-                                   keyColumns,
-                                   false
-                               );
-                             } else {
-                               frameWriterFactory = FrameWriters.makeColumnBasedFrameWriterFactory(
-                                   new SingleMemoryAllocatorFactory(allocator),
-                                   signature,
-                                   keyColumns
-                               );
-                             }
+    try (final CursorHolder cursorHolder = inputSegment.asStorageAdapter()
+                                                       .makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
 
-                             ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+      int numRows = 0;
+      final FrameWriterFactory frameWriterFactory;
+      if (FrameType.ROW_BASED.equals(outputFrameType)) {
+        frameWriterFactory = FrameWriters.makeRowBasedFrameWriterFactory(
+            new SingleMemoryAllocatorFactory(allocator),
+            signature,
+            keyColumns,
+            false
+        );
+      } else {
+        frameWriterFactory = FrameWriters.makeColumnBasedFrameWriterFactory(
+            new SingleMemoryAllocatorFactory(allocator),
+            signature,
+            keyColumns
+        );
+      }
 
-                             if (capabilitiesAdjustFn != null) {
-                               columnSelectorFactory = new OverrideCapabilitiesColumnSelectorFactory(
-                                   columnSelectorFactory,
-                                   capabilitiesAdjustFn
-                               );
-                             }
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-                             try (final FrameWriter frameWriter =
-                                      frameWriterFactory.newFrameWriter(columnSelectorFactory)) {
-                               while (!cursor.isDone() && frameWriter.addSelection()) {
-                                 numRows++;
-                                 cursor.advance();
-                               }
+      if (capabilitiesAdjustFn != null) {
+        columnSelectorFactory = new OverrideCapabilitiesColumnSelectorFactory(
+            columnSelectorFactory,
+            capabilitiesAdjustFn
+        );
+      }
 
-                               return Pair.of(Frame.wrap(frameWriter.toByteArray()), numRows);
-                             }
-                           }
-                       );
+      try (final FrameWriter frameWriter = frameWriterFactory.newFrameWriter(columnSelectorFactory)) {
+        while (!cursor.isDone() && frameWriter.addSelection()) {
+          numRows++;
+          cursor.advance();
+        }
+
+        return Pair.of(Frame.wrap(frameWriter.toByteArray()), numRows);
+      }
+    }
   }
 
   /**
diff --git a/processing/src/test/java/org/apache/druid/granularity/QueryGranularityTest.java b/processing/src/test/java/org/apache/druid/granularity/QueryGranularityTest.java
index dbb952fbf1c..41834d448fb 100644
--- a/processing/src/test/java/org/apache/druid/granularity/QueryGranularityTest.java
+++ b/processing/src/test/java/org/apache/druid/granularity/QueryGranularityTest.java
@@ -33,6 +33,8 @@ import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.granularity.GranularityType;
 import org.apache.druid.java.util.common.granularity.PeriodGranularity;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Days;
@@ -1021,6 +1023,39 @@ public class QueryGranularityTest
     );
   }
 
+  @Test
+  public void testToVirtualColumn()
+  {
+    final Granularity hour = new PeriodGranularity(new Period("PT1H"), null, null);
+    final DateTime origin = DateTimes.of("2012-01-02T05:00:00.000-08:00");
+    final DateTimeZone tz = DateTimes.inferTzFromString("America/Los_Angeles");
+    final Granularity hourWithOrigin = new PeriodGranularity(new Period("PT1H"), origin, tz);
+    final PeriodGranularity hourWithTz = new PeriodGranularity(new Period("PT1H"), null, tz);
+    final Granularity duration = new DurationGranularity(
+        new Period("PT12H5M").toStandardDuration().getMillis(),
+        origin
+    );
+
+    ExpressionVirtualColumn column = Granularities.toVirtualColumn(hour, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertEquals("timestamp_floor(__time,'PT1H')", column.getExpression());
+    column = Granularities.toVirtualColumn(hourWithOrigin, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertEquals(ColumnHolder.TIME_COLUMN_NAME, column.getExpression());
+    column = Granularities.toVirtualColumn(hourWithTz, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertEquals(ColumnHolder.TIME_COLUMN_NAME, column.getExpression());
+    column = Granularities.toVirtualColumn(duration, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertEquals(ColumnHolder.TIME_COLUMN_NAME, column.getExpression());
+    column = Granularities.toVirtualColumn(Granularities.NONE, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertEquals(ColumnHolder.TIME_COLUMN_NAME, column.getExpression());
+    column = Granularities.toVirtualColumn(Granularities.ALL, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertNull(column);
+    column = Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertEquals("timestamp_floor(__time,'PT1H')", column.getExpression());
+    column = Granularities.toVirtualColumn(Granularities.MINUTE, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertEquals("timestamp_floor(__time,'PT1M')", column.getExpression());
+    column = Granularities.toVirtualColumn(Granularities.FIFTEEN_MINUTE, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+    Assert.assertEquals("timestamp_floor(__time,'PT15M')", column.getExpression());
+  }
+
   private void assertBucketStart(final Granularity granularity, final DateTime in, final DateTime expectedInProperTz)
   {
     Assert.assertEquals(
diff --git a/processing/src/test/java/org/apache/druid/query/CursorGranularizerTest.java b/processing/src/test/java/org/apache/druid/query/CursorGranularizerTest.java
new file mode 100644
index 00000000000..8aeaf626070
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/query/CursorGranularizerTest.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.druid.data.input.ListBasedInputRow;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
+import org.apache.druid.segment.IndexBuilder;
+import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.incremental.IncrementalIndexSchema;
+import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class CursorGranularizerTest extends InitializedNullHandlingTest
+{
+  @Rule
+  public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  private StorageAdapter adapter;
+
+  @Before
+  public void setup() throws IOException
+  {
+    final RowSignature signature = RowSignature.builder().add("x", ColumnType.STRING).build();
+    final List<String> dims = ImmutableList.of("x");
+    final IncrementalIndexSchema schema =
+        IncrementalIndexSchema.builder()
+                              .withDimensionsSpec(DimensionsSpec.builder().useSchemaDiscovery(true).build())
+                              .withQueryGranularity(Granularities.MINUTE)
+                              .build();
+    IndexBuilder bob =
+        IndexBuilder.create()
+                    .schema(schema)
+                    .rows(
+                        ImmutableList.of(
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T00:00Z"),
+                                dims,
+                                ImmutableList.of("a")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T00:01Z"),
+                                dims,
+                                ImmutableList.of("b")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T00:02Z"),
+                                dims,
+                                ImmutableList.of("c")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T00:03Z"),
+                                dims,
+                                ImmutableList.of("d")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T01:00Z"),
+                                dims,
+                                ImmutableList.of("e")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T01:01Z"),
+                                dims,
+                                ImmutableList.of("f")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T03:04Z"),
+                                dims,
+                                ImmutableList.of("g")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T03:05Z"),
+                                dims,
+                                ImmutableList.of("h")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T03:15Z"),
+                                dims,
+                                ImmutableList.of("i")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T05:03Z"),
+                                dims,
+                                ImmutableList.of("j")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T06:00Z"),
+                                dims,
+                                ImmutableList.of("k")
+                            ),
+                            new ListBasedInputRow(
+                                signature,
+                                DateTimes.of("2024-01-01T09:01Z"),
+                                dims,
+                                ImmutableList.of("l")
+                            )
+                        )
+                    )
+                    .tmpDir(temporaryFolder.newFolder());
+
+    adapter = new QueryableIndexStorageAdapter(bob.buildMMappedIndex());
+  }
+
+  @Test
+  public void testGranularizeFullScan()
+  {
+    try (CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      CursorGranularizer granularizer = CursorGranularizer.create(
+          adapter,
+          cursor,
+          Granularities.HOUR,
+          adapter.getInterval(),
+          false
+      );
+
+      final ColumnSelectorFactory selectorFactory = cursor.getColumnSelectorFactory();
+      final ColumnValueSelector xSelector = selectorFactory.makeColumnValueSelector("x");
+      final Sequence<List<String>> theSequence =
+          Sequences.simple(granularizer.getBucketIterable())
+                   .map(bucketInterval -> {
+                     List<String> bucket = new ArrayList<>();
+                     if (!granularizer.advanceToBucket(bucketInterval)) {
+                       return bucket;
+                     }
+                     while (!cursor.isDone()) {
+                       bucket.add((String) xSelector.getObject());
+                       if (!granularizer.advanceCursorWithinBucket()) {
+                         break;
+                       }
+                     }
+                     return bucket;
+                   });
+
+      List<List<String>> granularized = theSequence.toList();
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of("a", "b", "c", "d"),
+              ImmutableList.of("e", "f"),
+              ImmutableList.of(),
+              ImmutableList.of("g", "h", "i"),
+              ImmutableList.of(),
+              ImmutableList.of("j"),
+              ImmutableList.of("k"),
+              ImmutableList.of(),
+              ImmutableList.of(),
+              ImmutableList.of("l")
+          ),
+          granularized
+      );
+    }
+  }
+
+  @Test
+  public void testGranularizeFullScanDescending()
+  {
+    final CursorBuildSpec descending = CursorBuildSpec.builder()
+                                                      .setPreferredOrdering(Cursors.descendingTimeOrder())
+                                                      .build();
+    try (CursorHolder cursorHolder = adapter.makeCursorHolder(descending)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      CursorGranularizer granularizer = CursorGranularizer.create(
+          adapter,
+          cursor,
+          Granularities.HOUR,
+          adapter.getInterval(),
+          true
+      );
+
+      final ColumnSelectorFactory selectorFactory = cursor.getColumnSelectorFactory();
+      final ColumnValueSelector xSelector = selectorFactory.makeColumnValueSelector("x");
+      final Sequence<List<String>> theSequence =
+          Sequences.simple(granularizer.getBucketIterable())
+                   .map(bucketInterval -> {
+                     List<String> bucket = new ArrayList<>();
+                     if (!granularizer.advanceToBucket(bucketInterval)) {
+                       return bucket;
+                     }
+                     while (!cursor.isDone()) {
+                       bucket.add((String) xSelector.getObject());
+                       if (!granularizer.advanceCursorWithinBucket()) {
+                         break;
+                       }
+                     }
+                     return bucket;
+                   });
+
+      List<List<String>> granularized = theSequence.toList();
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of("l"),
+              ImmutableList.of(),
+              ImmutableList.of(),
+              ImmutableList.of("k"),
+              ImmutableList.of("j"),
+              ImmutableList.of(),
+              ImmutableList.of("i", "h", "g"),
+              ImmutableList.of(),
+              ImmutableList.of("f", "e"),
+              ImmutableList.of("d", "c", "b", "a")
+          ),
+          granularized
+      );
+    }
+  }
+}
diff --git a/processing/src/test/java/org/apache/druid/query/QueryContextsTest.java b/processing/src/test/java/org/apache/druid/query/QueryContextsTest.java
index 554ca7b8f4a..64538f33d8d 100644
--- a/processing/src/test/java/org/apache/druid/query/QueryContextsTest.java
+++ b/processing/src/test/java/org/apache/druid/query/QueryContextsTest.java
@@ -43,7 +43,6 @@ public class QueryContextsTest
     final Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         new HashMap<>()
     );
     Assert.assertEquals(300_000, query.context().getDefaultTimeout());
@@ -55,7 +54,6 @@ public class QueryContextsTest
     Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         new HashMap<>()
     );
     Assert.assertEquals(300_000, query.context().getTimeout());
@@ -70,7 +68,6 @@ public class QueryContextsTest
     Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         ImmutableMap.of(QueryContexts.TIMEOUT_KEY, 1000)
     );
     Assert.assertEquals(1000, query.context().getTimeout());
@@ -87,7 +84,6 @@ public class QueryContextsTest
     Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         ImmutableMap.of(QueryContexts.TIMEOUT_KEY, 1000)
     );
 
@@ -102,7 +98,6 @@ public class QueryContextsTest
     Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         ImmutableMap.of(QueryContexts.MAX_SCATTER_GATHER_BYTES_KEY, 1000)
     );
 
@@ -115,7 +110,6 @@ public class QueryContextsTest
     Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         ImmutableMap.of(QueryContexts.SECONDARY_PARTITION_PRUNING_KEY, false)
     );
     Assert.assertFalse(query.context().isSecondaryPartitionPruningEnabled());
@@ -127,7 +121,6 @@ public class QueryContextsTest
     Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         ImmutableMap.of()
     );
     Assert.assertTrue(query.context().isSecondaryPartitionPruningEnabled());
@@ -213,7 +206,6 @@ public class QueryContextsTest
     new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         queryContext
     ).context().getTimeout();
   }
@@ -297,7 +289,6 @@ public class QueryContextsTest
     Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         ImmutableMap.of("e1", "FORCE",
                         "e2", "INVALID_ENUM"
         )
@@ -320,7 +311,6 @@ public class QueryContextsTest
     Query<?> query = new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))),
-        false,
         ImmutableMap.of(QueryContexts.CTX_EXECUTION_MODE, "SYNC", QueryContexts.CTX_EXECUTION_MODE + "_1", "ASYNC")
     );
 
diff --git a/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java
index 1228c42b17d..34cb3dbcdea 100644
--- a/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java
@@ -73,7 +73,6 @@ public class ReportTimelineMissingSegmentQueryRunnerTest
       super(
           new TableDataSource("datasource"),
           new MultipleSpecificSegmentSpec(Collections.emptyList()),
-          false,
           new HashMap<>()
       );
     }
diff --git a/processing/src/test/java/org/apache/druid/query/TestQuery.java b/processing/src/test/java/org/apache/druid/query/TestQuery.java
index 200151c200c..186b02444b5 100644
--- a/processing/src/test/java/org/apache/druid/query/TestQuery.java
+++ b/processing/src/test/java/org/apache/druid/query/TestQuery.java
@@ -31,9 +31,9 @@ public class TestQuery extends BaseQuery
   @Nullable
   private Set<String> requiredColumns;
 
-  public TestQuery(DataSource dataSource, QuerySegmentSpec querySegmentSpec, boolean descending, Map context)
+  public TestQuery(DataSource dataSource, QuerySegmentSpec querySegmentSpec, Map context)
   {
-    super(dataSource, querySegmentSpec, descending, context);
+    super(dataSource, querySegmentSpec, context);
   }
 
   @Override
@@ -72,7 +72,6 @@ public class TestQuery extends BaseQuery
     return new TestQuery(
         getDataSource(),
         getQuerySegmentSpec(),
-        isDescending(),
         BaseQuery.computeOverriddenContext(getContext(), contextOverride)
     );
   }
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryTest.java
index 2784d6b8918..e83eac25e30 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryTest.java
@@ -40,7 +40,9 @@ import org.apache.druid.query.groupby.orderby.OrderByColumnSpec;
 import org.apache.druid.query.ordering.StringComparators;
 import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
 import org.apache.druid.query.spec.QuerySegmentSpec;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
 import org.apache.druid.testing.InitializedNullHandlingTest;
@@ -162,6 +164,154 @@ public class GroupByQueryTest extends InitializedNullHandlingTest
     Assert.assertEquals(innerQuerySegmentSpec, BaseQuery.getQuerySegmentSpecForLookUp(query));
   }
 
+  @Test
+  public void testAsCursorBuildSpecAllGranularity()
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        new ExpressionVirtualColumn("v0", "concat(placement, 'foo')", ColumnType.STRING, ExprMacroTable.nil())
+    );
+    final LongSumAggregatorFactory longSum = new LongSumAggregatorFactory("idx", "index");
+    GroupByQuery query = GroupByQuery
+        .builder()
+        .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+        .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+        .setDimensions(
+            new DefaultDimensionSpec(QueryRunnerTestHelper.QUALITY_DIMENSION, "alias"),
+            new DefaultDimensionSpec(
+                QueryRunnerTestHelper.MARKET_DIMENSION,
+                "market",
+                ColumnType.STRING_ARRAY
+            ),
+            new DefaultDimensionSpec("v0", "v0", ColumnType.STRING)
+        )
+        .setVirtualColumns(virtualColumns)
+        .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, longSum)
+        .setGranularity(Granularities.ALL)
+        .setPostAggregatorSpecs(ImmutableList.of(new FieldAccessPostAggregator("x", "idx")))
+        .setLimitSpec(
+            new DefaultLimitSpec(
+                ImmutableList.of(new OrderByColumnSpec(
+                    "alias",
+                    OrderByColumnSpec.Direction.ASCENDING,
+                    StringComparators.LEXICOGRAPHIC
+                )),
+                100
+            )
+        )
+        .build();
+
+    final CursorBuildSpec buildSpec = GroupingEngine.makeCursorBuildSpec(query, null);
+    Assert.assertEquals(QueryRunnerTestHelper.FIRST_TO_THIRD.getIntervals().get(0), buildSpec.getInterval());
+    Assert.assertEquals(ImmutableList.of("quality", "market", "v0"), buildSpec.getGroupingColumns());
+    Assert.assertEquals(ImmutableList.of(QueryRunnerTestHelper.ROWS_COUNT, longSum), buildSpec.getAggregators());
+    Assert.assertEquals(virtualColumns, buildSpec.getVirtualColumns());
+  }
+
+  @Test
+  public void testAsCursorBuildSpecDayGranularity()
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        new ExpressionVirtualColumn("v0", "concat(placement, 'foo')", ColumnType.STRING, ExprMacroTable.nil())
+    );
+    final LongSumAggregatorFactory longSum = new LongSumAggregatorFactory("idx", "index");
+    GroupByQuery query = GroupByQuery
+        .builder()
+        .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+        .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+        .setDimensions(
+            new DefaultDimensionSpec(QueryRunnerTestHelper.QUALITY_DIMENSION, "alias"),
+            new DefaultDimensionSpec(
+                QueryRunnerTestHelper.MARKET_DIMENSION,
+                "market",
+                ColumnType.STRING_ARRAY
+            ),
+            new DefaultDimensionSpec("v0", "v0", ColumnType.STRING)
+        )
+        .setVirtualColumns(virtualColumns)
+        .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, longSum)
+        .setGranularity(Granularities.DAY)
+        .setPostAggregatorSpecs(ImmutableList.of(new FieldAccessPostAggregator("x", "idx")))
+        .setLimitSpec(
+            new DefaultLimitSpec(
+                ImmutableList.of(new OrderByColumnSpec(
+                    "alias",
+                    OrderByColumnSpec.Direction.ASCENDING,
+                    StringComparators.LEXICOGRAPHIC
+                )),
+                100
+            )
+        )
+        .build();
+
+    final CursorBuildSpec buildSpec = GroupingEngine.makeCursorBuildSpec(query, null);
+    Assert.assertEquals(QueryRunnerTestHelper.FIRST_TO_THIRD.getIntervals().get(0), buildSpec.getInterval());
+    Assert.assertEquals(
+        ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "quality", "market", "v0"),
+        buildSpec.getGroupingColumns()
+    );
+    Assert.assertEquals(ImmutableList.of(QueryRunnerTestHelper.ROWS_COUNT, longSum), buildSpec.getAggregators());
+    Assert.assertEquals(
+        VirtualColumns.create(
+            Granularities.toVirtualColumn(query.getGranularity(), Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME),
+            virtualColumns.getVirtualColumns()[0]
+        ),
+        buildSpec.getVirtualColumns()
+    );
+  }
+
+  @Test
+  public void testAsCursorBuildSpecDayGranularityNameConflict()
+  {
+    // make conflicting column name to force artificial granularity column to have a different name
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        new ExpressionVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "concat(placement, 'foo')", ColumnType.STRING, ExprMacroTable.nil())
+    );
+    final LongSumAggregatorFactory longSum = new LongSumAggregatorFactory("idx", "index");
+    GroupByQuery query = GroupByQuery
+        .builder()
+        .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+        .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+        .setDimensions(
+            new DefaultDimensionSpec(QueryRunnerTestHelper.QUALITY_DIMENSION, "alias"),
+            new DefaultDimensionSpec(
+                QueryRunnerTestHelper.MARKET_DIMENSION,
+                "market",
+                ColumnType.STRING_ARRAY
+            ),
+            new DefaultDimensionSpec("v0", "v0", ColumnType.STRING)
+        )
+        .setVirtualColumns(virtualColumns)
+        .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, longSum)
+        .setGranularity(Granularities.DAY)
+        .setPostAggregatorSpecs(ImmutableList.of(new FieldAccessPostAggregator("x", "idx")))
+        .setLimitSpec(
+            new DefaultLimitSpec(
+                ImmutableList.of(new OrderByColumnSpec(
+                    "alias",
+                    OrderByColumnSpec.Direction.ASCENDING,
+                    StringComparators.LEXICOGRAPHIC
+                )),
+                100
+            )
+        )
+        .build();
+
+    final CursorBuildSpec buildSpec = GroupingEngine.makeCursorBuildSpec(query, null);
+    Assert.assertEquals(QueryRunnerTestHelper.FIRST_TO_THIRD.getIntervals().get(0), buildSpec.getInterval());
+    Assert.assertEquals(
+        ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + "0", "quality", "market", "v0"),
+        buildSpec.getGroupingColumns()
+    );
+    Assert.assertEquals(ImmutableList.of(QueryRunnerTestHelper.ROWS_COUNT, longSum), buildSpec.getAggregators());
+    Assert.assertEquals(
+        VirtualColumns.create(
+            Granularities.toVirtualColumn(query.getGranularity(), Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + "0"),
+            virtualColumns.getVirtualColumns()[0]
+        ),
+        buildSpec.getVirtualColumns()
+    );
+  }
+
   @Test
   public void testEquals()
   {
@@ -176,7 +326,8 @@ public class GroupByQueryTest extends InitializedNullHandlingTest
                       "forceLimitPushDown",
                       "postProcessingFn",
                       "resultRowSignature",
-                      "universalTimestamp"
+                      "universalTimestamp",
+                      "groupingColumns"
                   )
                   .verify();
   }
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java b/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java
index 6c0ff3b4951..adbcbc187bb 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java
@@ -34,9 +34,10 @@ import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.expression.TestExprMacroTable;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.InDimFilter;
 import org.apache.druid.query.filter.SelectorDimFilter;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
@@ -654,18 +655,17 @@ public class NestedDataGroupByQueryTest extends InitializedNullHandlingTest
     List<Segment> segments = segmentsGenerator.apply(tempFolder, closer);
     Supplier<List<ResultRow>> runner =
         () -> helper.runQueryOnSegmentsObjs(segments, groupQuery).toList();
-    Filter filter = groupQuery.getFilter() == null ? null : groupQuery.getFilter().toFilter();
+    final CursorBuildSpec spec = GroupingEngine.makeCursorBuildSpec(groupQuery, null);
     boolean allCanVectorize = segments.stream()
                                       .allMatch(
-                                          s -> s.asStorageAdapter()
-                                                .canVectorize(
-                                                    filter,
-                                                    groupQuery.getVirtualColumns(),
-                                                    groupQuery.isDescending()
-                                                )
-                                      );
+                                          s -> {
+                                            final CursorHolder cursorHolder = s.asStorageAdapter()
+                                                                               .makeCursorHolder(spec);
+                                            final boolean canVectorize = cursorHolder.canVectorize();
+                                            cursorHolder.close();
+                                            return canVectorize;
+                                          });
 
-    Assert.assertEquals(NestedDataTestUtils.expectSegmentGeneratorCanVectorize(segmentsName), allCanVectorize);
     if (!allCanVectorize) {
       if (vectorize == QueryContexts.Vectorize.FORCE) {
         Throwable t = Assert.assertThrows(RuntimeException.class, runner::get);
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngineIteratorTest.java b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngineIteratorTest.java
index 8bc83277c65..ce38427728c 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngineIteratorTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngineIteratorTest.java
@@ -27,13 +27,14 @@ import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
 import org.apache.druid.query.groupby.GroupByQueryRunnerTest;
+import org.apache.druid.query.groupby.GroupingEngine;
 import org.apache.druid.query.groupby.epinephelinae.VectorGrouper;
 import org.apache.druid.query.groupby.epinephelinae.vector.VectorGroupByEngine.VectorGroupByEngineIterator;
 import org.apache.druid.segment.ColumnProcessors;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.TestIndex;
-import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.segment.vector.VectorCursor;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.joda.time.Interval;
@@ -61,15 +62,11 @@ public class VectorGroupByEngineIteratorTest extends InitializedNullHandlingTest
         .setAggregatorSpecs(factory)
         .build();
     final StorageAdapter storageAdapter = new QueryableIndexStorageAdapter(TestIndex.getMMappedTestIndex());
-    final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[4096]);
-    final VectorCursor cursor = storageAdapter.makeVectorCursor(
-        Filters.toFilter(query.getDimFilter()),
-        interval,
-        query.getVirtualColumns(),
-        false,
-        query.context().getVectorSize(),
-        null
+    final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(
+        GroupingEngine.makeCursorBuildSpec(query, null)
     );
+    final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[4096]);
+    final VectorCursor cursor = cursorHolder.asVectorCursor();
     final List<GroupByVectorColumnSelector> dimensions = query.getDimensions().stream().map(
         dimensionSpec ->
             ColumnProcessors.makeVectorProcessor(
@@ -100,5 +97,6 @@ public class VectorGroupByEngineIteratorTest extends InitializedNullHandlingTest
     };
     iterator.close();
     Mockito.verify(grouperCaptor.getValue()).close();
+    cursorHolder.close();
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/query/lookup/LookupSegmentTest.java b/processing/src/test/java/org/apache/druid/query/lookup/LookupSegmentTest.java
index 2d36ffa1e63..5826301bcd7 100644
--- a/processing/src/test/java/org/apache/druid/query/lookup/LookupSegmentTest.java
+++ b/processing/src/test/java/org/apache/druid/query/lookup/LookupSegmentTest.java
@@ -24,14 +24,13 @@ import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.Lists;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.extraction.MapLookupExtractor;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.RowBasedStorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.timeline.SegmentId;
@@ -190,47 +189,38 @@ public class LookupSegmentTest
   }
 
   @Test
-  public void test_asStorageAdapter_makeCursors()
+  public void test_asStorageAdapter_makeCursor()
   {
-    final Sequence<Cursor> cursors = LOOKUP_SEGMENT.asStorageAdapter().makeCursors(
-        null,
-        Intervals.of("1970/PT1H"),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setInterval(Intervals.of("1970/PT1H"))
+                                                     .build();
+    try (final CursorHolder cursorHolder = LOOKUP_SEGMENT.asStorageAdapter().makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
 
-    final List<Pair<String, String>> kvs = new ArrayList<>();
+      final List<Pair<String, String>> kvs = new ArrayList<>();
 
-    cursors.accumulate(
-        null,
-        (ignored, cursor) -> {
-          final ColumnValueSelector keySelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("k");
-          final ColumnValueSelector valueSelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
+      final ColumnValueSelector keySelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("k");
+      final ColumnValueSelector valueSelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
 
-          while (!cursor.isDone()) {
-            kvs.add(Pair.of(String.valueOf(keySelector.getObject()), String.valueOf(valueSelector.getObject())));
-            cursor.advanceUninterruptibly();
-          }
+      while (!cursor.isDone()) {
+        kvs.add(Pair.of(String.valueOf(keySelector.getObject()), String.valueOf(valueSelector.getObject())));
+        cursor.advanceUninterruptibly();
+      }
 
-          return null;
-        }
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            Pair.of("a", "b"),
-            Pair.of("x", "y")
-        ),
-        kvs
-    );
+      Assert.assertEquals(
+          ImmutableList.of(
+              Pair.of("a", "b"),
+              Pair.of("x", "y")
+          ),
+          kvs
+      );
+    }
   }
 
   @Test
   public void test_asStorageAdapter_isRowBasedAdapter()
   {
-    // This allows us to assume that RowBasedStorageAdapterTest is further exercising makeCursors and verifying misc.
+    // This allows us to assume that RowBasedStorageAdapterTest is further exercising makeCursor and verifying misc.
     // methods like getMinTime, getMaxTime, getMetadata, etc, without checking them explicitly in _this_ test class.
     Assert.assertThat(LOOKUP_SEGMENT.asStorageAdapter(), CoreMatchers.instanceOf(RowBasedStorageAdapter.class));
   }
diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java
index e919799deb5..8e39a640acb 100644
--- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java
@@ -205,15 +205,11 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest
                       .merge(true)
                       .build();
 
-    int preferedSize1 = 0;
-    int placementSize2 = 0;
-    int overallSize1 = 153543;
-    int overallSize2 = 153543;
+    int placementSize = 0;
+    int overallSize = 153543;
     if (bitmaps) {
-      preferedSize1 = mmap1 ? 10881 : 10764;
-      placementSize2 = mmap2 ? 10881 : 0;
-      overallSize1 = mmap1 ? 201345 : 200831;
-      overallSize2 = mmap2 ? 201345 : 200831;
+      placementSize = 10881;
+      overallSize = 201345;
     }
 
     final Map<String, AggregatorFactory> expectedAggregators = new HashMap<>();
@@ -256,7 +252,7 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest
                     ValueType.STRING.toString(),
                     false,
                     false,
-                    preferedSize1,
+                    placementSize,
                     1,
                     "preferred",
                     "preferred",
@@ -264,7 +260,7 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest
                 )
             )
         ),
-        overallSize1,
+        overallSize,
         1209,
         expectedAggregators,
         null,
@@ -306,7 +302,7 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest
                     ValueType.STRING.toString(),
                     false,
                     false,
-                    placementSize2,
+                    placementSize,
                     1,
                     null,
                     null,
@@ -314,8 +310,7 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest
                 )
             )
         ),
-        // null_column will be included only for incremental index, which makes a little bigger result than expected
-        overallSize2,
+        overallSize,
         1209,
         expectedAggregators,
         null,
@@ -558,18 +553,16 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest
   @Test
   public void testSegmentMetadataQueryWithDefaultAnalysisMerge()
   {
-    int size1 = 0;
-    int size2 = 0;
+    int size = 0;
     if (bitmaps) {
-      size1 = mmap1 ? 10881 : 10764;
-      size2 = mmap2 ? 10881 : 10764;
+      size = 10881;
     }
     ColumnAnalysis analysis = new ColumnAnalysis(
         ColumnType.STRING,
         ValueType.STRING.toString(),
         false,
         false,
-        size1 + size2,
+        size * 2,
         1,
         "preferred",
         "preferred",
@@ -581,18 +574,16 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest
   @Test
   public void testSegmentMetadataQueryWithDefaultAnalysisMerge2()
   {
-    int size1 = 0;
-    int size2 = 0;
+    int size = 0;
     if (bitmaps) {
-      size1 = mmap1 ? 6882 : 6808;
-      size2 = mmap2 ? 6882 : 6808;
+      size = 6882;
     }
     ColumnAnalysis analysis = new ColumnAnalysis(
         ColumnType.STRING,
         ValueType.STRING.toString(),
         false,
         false,
-        size1 + size2,
+        size * 2,
         3,
         "spot",
         "upfront",
@@ -604,18 +595,16 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest
   @Test
   public void testSegmentMetadataQueryWithDefaultAnalysisMerge3()
   {
-    int size1 = 0;
-    int size2 = 0;
+    int size = 0;
     if (bitmaps) {
-      size1 = mmap1 ? 9765 : 9660;
-      size2 = mmap2 ? 9765 : 9660;
+      size = 9765;
     }
     ColumnAnalysis analysis = new ColumnAnalysis(
         ColumnType.STRING,
         ValueType.STRING.toString(),
         false,
         false,
-        size1 + size2,
+        size * 2,
         9,
         "automotive",
         "travel",
diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java
index 9f37a47aae4..b1db91a9cb8 100644
--- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java
@@ -109,7 +109,7 @@ public class SegmentMetadataUnionQueryTest extends InitializedNullHandlingTest
                     ValueType.STRING.toString(),
                     false,
                     false,
-                    mmap ? 43524 : 43056,
+                    43524,
                     1,
                     "preferred",
                     "preferred",
@@ -117,7 +117,7 @@ public class SegmentMetadataUnionQueryTest extends InitializedNullHandlingTest
                 )
             )
         ),
-        mmap ? 805380 : 803324,
+        805380,
         4836,
         null,
         null,
diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/StorageAdapterRowsAndColumnsTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/StorageAdapterRowsAndColumnsTest.java
index bb2826498c5..4395ce9dd5b 100644
--- a/processing/src/test/java/org/apache/druid/query/rowsandcols/StorageAdapterRowsAndColumnsTest.java
+++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/StorageAdapterRowsAndColumnsTest.java
@@ -32,13 +32,13 @@ public class StorageAdapterRowsAndColumnsTest extends RowsAndColumnsTestBase
     super(StorageAdapterRowsAndColumns.class);
   }
 
-  public static Function<MapOfColumnsRowsAndColumns, StorageAdapterRowsAndColumns> MAKER = input -> {
-    return buildFrame(input);
-  };
+  public static Function<MapOfColumnsRowsAndColumns, StorageAdapterRowsAndColumns> MAKER =
+      StorageAdapterRowsAndColumnsTest::buildFrame;
 
   private static StorageAdapterRowsAndColumns buildFrame(MapOfColumnsRowsAndColumns input)
   {
-    ColumnBasedFrameRowsAndColumns fRAC = ColumnBasedFrameRowsAndColumnsTest.buildFrame(input);
-    return new StorageAdapterRowsAndColumns(fRAC.as(StorageAdapter.class));
+    try (ColumnBasedFrameRowsAndColumns fRAC = ColumnBasedFrameRowsAndColumnsTest.buildFrame(input)) {
+      return new StorageAdapterRowsAndColumns(fRAC.as(StorageAdapter.class));
+    }
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/semantic/RowsAndColumnsDecoratorTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/semantic/RowsAndColumnsDecoratorTest.java
index 919f9fcff82..023a6294ab9 100644
--- a/processing/src/test/java/org/apache/druid/query/rowsandcols/semantic/RowsAndColumnsDecoratorTest.java
+++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/semantic/RowsAndColumnsDecoratorTest.java
@@ -24,8 +24,6 @@ import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.UOE;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.InDimFilter;
 import org.apache.druid.query.groupby.ResultRow;
@@ -37,7 +35,8 @@ import org.apache.druid.query.rowsandcols.column.ColumnAccessor;
 import org.apache.druid.segment.ArrayListSegment;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.TypeStrategy;
@@ -250,28 +249,21 @@ public class RowsAndColumnsDecoratorTest extends SemanticTestBase
           },
           siggy
       );
-      final Sequence<Cursor> cursors = seggy
-          .asStorageAdapter()
-          .makeCursors(
-              filter,
-              interval == null ? Intervals.ETERNITY : interval,
-              VirtualColumns.EMPTY,
-              Granularities.ALL,
-              false,
-              null
-          );
+      final CursorBuildSpec.CursorBuildSpecBuilder builder = CursorBuildSpec.builder()
+                                                                            .setFilter(filter);
+      if (interval != null) {
+        builder.setInterval(interval);
+      }
+      try (final CursorHolder cursorHolder = seggy.asStorageAdapter().makeCursorHolder(builder.build())) {
+        final Cursor cursor = cursorHolder.asCursor();
 
-      vals = cursors.accumulate(
-          new ArrayList<>(),
-          (accumulated, in) -> {
-            final ColumnValueSelector idSupplier = in.getColumnSelectorFactory().makeColumnValueSelector("arrayIndex");
-            while (!in.isDone()) {
-              accumulated.add(originalVals[(int) idSupplier.getLong()]);
-              in.advance();
-            }
-            return accumulated;
-          }
-      );
+        vals = new ArrayList<>();
+        final ColumnValueSelector idSupplier = cursor.getColumnSelectorFactory().makeColumnValueSelector("arrayIndex");
+        while (!cursor.isDone()) {
+          vals.add(originalVals[(int) idSupplier.getLong()]);
+          cursor.advance();
+        }
+      }
     }
 
     if (ordering != null) {
diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryLimitRowIteratorTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryLimitRowIteratorTest.java
index 60b098b43bf..e187ddfd630 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryLimitRowIteratorTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryLimitRowIteratorTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryRunnerTestHelper;
 import org.apache.druid.query.context.ResponseContext;
@@ -104,7 +105,7 @@ public class ScanQueryLimitRowIteratorTest
   {
     ScanQuery query = Druids.newScanQueryBuilder()
                             .limit(limit)
-                            .order(ScanQuery.Order.NONE)
+                            .order(Order.NONE)
                             .dataSource("some datasource")
                             .batchSize(batchSize)
                             .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -145,7 +146,7 @@ public class ScanQueryLimitRowIteratorTest
   {
     ScanQuery query = Druids.newScanQueryBuilder()
                             .limit(limit)
-                            .order(ScanQuery.Order.DESCENDING)
+                            .order(Order.DESCENDING)
                             .dataSource("some datasource")
                             .batchSize(batchSize)
                             .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -184,7 +185,7 @@ public class ScanQueryLimitRowIteratorTest
   {
     ScanQuery query = Druids.newScanQueryBuilder()
                             .limit(limit)
-                            .order(ScanQuery.Order.DESCENDING)
+                            .order(Order.DESCENDING)
                             .dataSource("some datasource")
                             .batchSize(batchSize)
                             .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryResultOrderingTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryResultOrderingTest.java
index 5b155aa4a98..2bfd47fb699 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryResultOrderingTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryResultOrderingTest.java
@@ -32,6 +32,7 @@ import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.query.DefaultGenericQueryMetricsFactory;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryRunner;
 import org.apache.druid.query.context.ResponseContext;
@@ -42,6 +43,7 @@ import org.apache.druid.segment.RowBasedSegment;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.apache.druid.timeline.SegmentId;
 import org.joda.time.DateTime;
 import org.junit.Assert;
@@ -65,7 +67,7 @@ import java.util.stream.IntStream;
  * Ensures that we have run-to-run stability of result order, which is important for offset-based pagination.
  */
 @RunWith(Parameterized.class)
-public class ScanQueryResultOrderingTest
+public class ScanQueryResultOrderingTest extends InitializedNullHandlingTest
 {
   private static final String DATASOURCE = "datasource";
   private static final String ID_COLUMN = "id";
@@ -206,7 +208,7 @@ public class ScanQueryResultOrderingTest
               .dataSource("ds")
               .intervals(new MultipleIntervalSegmentSpec(Collections.singletonList(Intervals.of("2000/P1D"))))
               .columns(ColumnHolder.TIME_COLUMN_NAME, ID_COLUMN)
-              .order(ScanQuery.Order.NONE)
+              .order(Order.NONE)
               .build(),
         ImmutableList.of(
             101,
@@ -240,7 +242,7 @@ public class ScanQueryResultOrderingTest
               .dataSource("ds")
               .intervals(new MultipleIntervalSegmentSpec(Collections.singletonList(Intervals.of("2000/P1D"))))
               .columns(ColumnHolder.TIME_COLUMN_NAME, ID_COLUMN)
-              .order(ScanQuery.Order.ASCENDING)
+              .order(Order.ASCENDING)
               .build(),
         ImmutableList.of(
             101,
@@ -274,7 +276,7 @@ public class ScanQueryResultOrderingTest
               .dataSource("ds")
               .intervals(new MultipleIntervalSegmentSpec(Collections.singletonList(Intervals.of("2000/P1D"))))
               .columns(ColumnHolder.TIME_COLUMN_NAME, ID_COLUMN)
-              .order(ScanQuery.Order.DESCENDING)
+              .order(Order.DESCENDING)
               .build(),
         ImmutableList.of(
             8,
diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerFactoryTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerFactoryTest.java
index 55c23a8d8c9..794656198d0 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerFactoryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerFactoryTest.java
@@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.query.DefaultGenericQueryMetricsFactory;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryRunner;
 import org.apache.druid.query.QueryRunnerTestHelper;
@@ -91,7 +92,7 @@ public class ScanQueryRunnerFactoryTest
         final int batchSize,
         final long limit,
         final ScanQuery.ResultFormat resultFormat,
-        final ScanQuery.Order order
+        final Order order
     )
     {
       this.numElements = numElements;
@@ -116,9 +117,9 @@ public class ScanQueryRunnerFactoryTest
           ScanQuery.ResultFormat.RESULT_FORMAT_LIST,
           ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST
       );
-      List<ScanQuery.Order> order = ImmutableList.of(
-          ScanQuery.Order.ASCENDING,
-          ScanQuery.Order.DESCENDING
+      List<Order> order = ImmutableList.of(
+          Order.ASCENDING,
+          Order.DESCENDING
       );
 
       return QueryRunnerTestHelper.cartesian(
@@ -147,7 +148,7 @@ public class ScanQueryRunnerFactoryTest
         } else if (o1 < o2) {
           retVal = -1;
         }
-        if (query.getTimeOrder().equals(ScanQuery.Order.DESCENDING)) {
+        if (query.getTimeOrder().equals(Order.DESCENDING)) {
           return retVal * -1;
         }
         return retVal;
@@ -200,7 +201,7 @@ public class ScanQueryRunnerFactoryTest
         scanResultValues3.add(ScanQueryTestHelper.generateScanResultValue(timestamp, resultFormat, 1));
       }
 
-      if (query.getTimeOrder() == ScanQuery.Order.DESCENDING) {
+      if (query.getTimeOrder() == Order.DESCENDING) {
         Collections.reverse(scanResultValues1);
         Collections.reverse(scanResultValues2);
         Collections.reverse(scanResultValues3);
@@ -221,7 +222,7 @@ public class ScanQueryRunnerFactoryTest
 
       List<List<QueryRunner<ScanResultValue>>> groupedRunners = new ArrayList<>(2);
 
-      if (query.getTimeOrder() == ScanQuery.Order.DESCENDING) {
+      if (query.getTimeOrder() == Order.DESCENDING) {
         groupedRunners.add(Arrays.asList(runnerSegment2Partition1, runnerSegment2Partition2));
         groupedRunners.add(Arrays.asList(runnerSegment1Partition1, runnerSegment1Partition2));
       } else {
@@ -236,7 +237,7 @@ public class ScanQueryRunnerFactoryTest
         } else if (o1 < o2) {
           retVal = -1;
         }
-        if (query.getTimeOrder().equals(ScanQuery.Order.DESCENDING)) {
+        if (query.getTimeOrder().equals(Order.DESCENDING)) {
           return retVal * -1;
         }
         return retVal;
@@ -268,7 +269,7 @@ public class ScanQueryRunnerFactoryTest
 
       // check ordering is correct
       for (int i = 1; i < output.size(); i++) {
-        if (query.getTimeOrder().equals(ScanQuery.Order.DESCENDING)) {
+        if (query.getTimeOrder().equals(Order.DESCENDING)) {
           Assert.assertTrue(output.get(i).getFirstEventTimestamp(resultFormat) <
                             output.get(i - 1).getFirstEventTimestamp(resultFormat));
         } else {
@@ -370,7 +371,7 @@ public class ScanQueryRunnerFactoryTest
                                      .collect(Collectors.toList())
                         )
                     )
-                    .order(ScanQuery.Order.ASCENDING)
+                    .order(Order.ASCENDING)
                     .build()
           ),
           ResponseContext.createEmpty()
diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerTest.java
index f2c3a9ea18f..8dd18390618 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerTest.java
@@ -41,6 +41,7 @@ import org.apache.druid.query.DefaultGenericQueryMetricsFactory;
 import org.apache.druid.query.DirectQueryProcessingPool;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.MetricsEmittingQueryRunner;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryRunner;
@@ -535,7 +536,7 @@ public class ScanQueryRunnerTest extends InitializedNullHandlingTest
               QueryRunnerTestHelper.INDEX_METRIC
           )
           .limit(limit)
-          .order(ScanQuery.Order.ASCENDING)
+          .order(Order.ASCENDING)
           .context(ImmutableMap.of(ScanQuery.CTX_KEY_OUTERMOST, false))
           .build();
 
@@ -607,7 +608,7 @@ public class ScanQueryRunnerTest extends InitializedNullHandlingTest
               QueryRunnerTestHelper.INDEX_METRIC
           )
           .limit(limit)
-          .order(ScanQuery.Order.DESCENDING)
+          .order(Order.DESCENDING)
           .build();
 
       Iterable<ScanResultValue> results = runner.run(QueryPlus.wrap(query)).toList();
@@ -703,7 +704,7 @@ public class ScanQueryRunnerTest extends InitializedNullHandlingTest
               QueryRunnerTestHelper.INDEX_METRIC
           )
           .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
-          .order(ScanQuery.Order.ASCENDING)
+          .order(Order.ASCENDING)
           .limit(limit)
           .build();
 
@@ -776,7 +777,7 @@ public class ScanQueryRunnerTest extends InitializedNullHandlingTest
               QueryRunnerTestHelper.INDEX_METRIC
           )
           .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
-          .order(ScanQuery.Order.DESCENDING)
+          .order(Order.DESCENDING)
           .context(ImmutableMap.of(ScanQuery.CTX_KEY_OUTERMOST, false))
           .limit(limit)
           .build();
diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanQuerySpecTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanQuerySpecTest.java
index e5069743a9e..f5ce99e5a3e 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/ScanQuerySpecTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/ScanQuerySpecTest.java
@@ -22,6 +22,8 @@ package org.apache.druid.query.scan;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryRunnerTestHelper;
 import org.apache.druid.query.TableDataSource;
@@ -57,7 +59,7 @@ public class ScanQuerySpecTest
         0,
         0,
         3,
-        ScanQuery.Order.NONE,
+        Order.NONE,
         null,
         null,
         Arrays.asList("market", "quality", "index"),
@@ -85,7 +87,7 @@ public class ScanQuerySpecTest
         0,
         0,
         3,
-        ScanQuery.Order.ASCENDING,
+        Order.ASCENDING,
         null,
         null,
         Arrays.asList("market", "quality", "index", "__time"),
@@ -96,9 +98,9 @@ public class ScanQuerySpecTest
     String serializedJson = JSON_MAPPER.writeValueAsString(expectedQuery);
     Assert.assertEquals(originalJson, serializedJson);
     Assert.assertEquals(expectedQuery, JSON_MAPPER.readValue(originalJson, ScanQuery.class));
-    Assert.assertEquals(ScanQuery.Order.ASCENDING, expectedQuery.getTimeOrder());
+    Assert.assertEquals(Order.ASCENDING, expectedQuery.getTimeOrder());
     Assert.assertEquals(
-        Collections.singletonList(new ScanQuery.OrderBy("__time", ScanQuery.Order.ASCENDING)),
+        Collections.singletonList(OrderBy.ascending("__time")),
         expectedQuery.getOrderBys()
     );
   }
@@ -118,7 +120,7 @@ public class ScanQuerySpecTest
         0,
         3,
         null,
-        Collections.singletonList(new ScanQuery.OrderBy("quality", ScanQuery.Order.ASCENDING)),
+        Collections.singletonList(OrderBy.ascending("quality")),
         null,
         Arrays.asList("market", "quality", "index", "__time"),
         null,
@@ -128,9 +130,9 @@ public class ScanQuerySpecTest
     String serializedJson = JSON_MAPPER.writeValueAsString(expectedQuery);
     Assert.assertEquals(originalJson, serializedJson);
     Assert.assertEquals(expectedQuery, JSON_MAPPER.readValue(originalJson, ScanQuery.class));
-    Assert.assertEquals(ScanQuery.Order.NONE, expectedQuery.getTimeOrder());
+    Assert.assertEquals(Order.NONE, expectedQuery.getTimeOrder());
     Assert.assertEquals(
-        Collections.singletonList(new ScanQuery.OrderBy("quality", ScanQuery.Order.ASCENDING)),
+        Collections.singletonList(OrderBy.ascending("quality")),
         expectedQuery.getOrderBys()
     );
   }
@@ -146,7 +148,7 @@ public class ScanQuerySpecTest
         0,
         1,
         3,
-        ScanQuery.Order.NONE,
+        Order.NONE,
         null,
         null,
         Arrays.asList("market", "quality", "index"),
diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryTest.java
index b0d3432c0db..f5a784b3841 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryTest.java
@@ -27,14 +27,22 @@ import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
+import org.apache.druid.query.QueryRunnerTestHelper;
 import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
 import org.apache.druid.query.spec.QuerySegmentSpec;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
+import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.joda.time.Interval;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -47,7 +55,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
 
-public class ScanQueryTest
+public class ScanQueryTest extends InitializedNullHandlingTest
 {
   private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper();
   private static QuerySegmentSpec intervalSpec;
@@ -117,7 +125,7 @@ public class ScanQueryTest
   public void testAscendingScanQueryWithInvalidColumns()
   {
     Druids.newScanQueryBuilder()
-          .order(ScanQuery.Order.ASCENDING)
+          .order(Order.ASCENDING)
           .columns(ImmutableList.of("not time", "also not time"))
           .dataSource("source")
           .intervals(intervalSpec)
@@ -128,7 +136,7 @@ public class ScanQueryTest
   public void testDescendingScanQueryWithInvalidColumns()
   {
     Druids.newScanQueryBuilder()
-          .order(ScanQuery.Order.DESCENDING)
+          .order(Order.DESCENDING)
           .columns(ImmutableList.of("not time", "also not time"))
           .dataSource("source")
           .intervals(intervalSpec)
@@ -143,12 +151,12 @@ public class ScanQueryTest
         IllegalArgumentException.class,
         () ->
             Druids.newScanQueryBuilder()
-                  .order(ScanQuery.Order.ASCENDING)
+                  .order(Order.ASCENDING)
                   .orderBy(
                       // Not ok, even though it starts with __time ASC, because it also has non-time component.
                       ImmutableList.of(
-                          new ScanQuery.OrderBy("__time", ScanQuery.Order.ASCENDING),
-                          new ScanQuery.OrderBy("quality", ScanQuery.Order.DESCENDING)
+                          OrderBy.ascending("__time"),
+                          OrderBy.descending("quality")
                       )
                   )
                   .columns(ImmutableList.of("__time", "quality"))
@@ -163,8 +171,8 @@ public class ScanQueryTest
   {
     Assert.assertNotNull(
         Druids.newScanQueryBuilder()
-              .order(ScanQuery.Order.ASCENDING)
-              .orderBy(ImmutableList.of(new ScanQuery.OrderBy("__time", ScanQuery.Order.ASCENDING)))
+              .order(Order.ASCENDING)
+              .orderBy(ImmutableList.of(OrderBy.ascending("__time")))
               .columns(ImmutableList.of("__time", "quality"))
               .dataSource("source")
               .intervals(intervalSpec)
@@ -176,9 +184,9 @@ public class ScanQueryTest
   @Test
   public void testValidScanQueryInitialization()
   {
-    List<ScanQuery.Order> nonOrderedOrders = Arrays.asList(null, ScanQuery.Order.NONE);
+    List<Order> nonOrderedOrders = Arrays.asList(null, Order.NONE);
 
-    for (ScanQuery.Order order : nonOrderedOrders) {
+    for (Order order : nonOrderedOrders) {
       Druids.newScanQueryBuilder()
             .order(order)
             .columns(ImmutableList.of("not time"))
@@ -208,9 +216,9 @@ public class ScanQueryTest
             .build();
     }
 
-    Set<ScanQuery.Order> orderedOrders = ImmutableSet.of(ScanQuery.Order.ASCENDING, ScanQuery.Order.DESCENDING);
+    Set<Order> orderedOrders = ImmutableSet.of(Order.ASCENDING, Order.DESCENDING);
 
-    for (ScanQuery.Order order : orderedOrders) {
+    for (Order order : orderedOrders) {
       Druids.newScanQueryBuilder()
             .order(order)
             .columns((List<String>) null)
@@ -246,7 +254,7 @@ public class ScanQueryTest
   {
     // Should be able to handle merging s1, s2, s3
     ScanQuery noOrderScan = Druids.newScanQueryBuilder()
-                                  .order(ScanQuery.Order.NONE)
+                                  .order(Order.NONE)
                                   .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
                                   .dataSource("some src")
                                   .intervals(intervalSpec)
@@ -254,7 +262,7 @@ public class ScanQueryTest
 
     // Should only handle s1 and s2
     ScanQuery descendingOrderScan = Druids.newScanQueryBuilder()
-                                          .order(ScanQuery.Order.DESCENDING)
+                                          .order(Order.DESCENDING)
                                           .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
                                           .dataSource("some src")
                                           .intervals(intervalSpec)
@@ -262,7 +270,7 @@ public class ScanQueryTest
 
     // Should only handle s1 and s2
     ScanQuery ascendingOrderScan = Druids.newScanQueryBuilder()
-                                         .order(ScanQuery.Order.ASCENDING)
+                                         .order(Order.ASCENDING)
                                          .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
                                          .dataSource("some src")
                                          .intervals(intervalSpec)
@@ -311,7 +319,7 @@ public class ScanQueryTest
   public void testTimeOrderingWithoutTimeColumn()
   {
     ScanQuery descendingOrderScan = Druids.newScanQueryBuilder()
-                                          .order(ScanQuery.Order.DESCENDING)
+                                          .order(Order.DESCENDING)
                                           .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
                                           .dataSource("some src")
                                           .intervals(intervalSpec)
@@ -334,7 +342,7 @@ public class ScanQueryTest
     final ScanQuery scanQuery =
         Druids.newScanQueryBuilder()
               .columns("__time")
-              .orderBy(Collections.singletonList(new ScanQuery.OrderBy("__time", ScanQuery.Order.DESCENDING)))
+              .orderBy(Collections.singletonList(OrderBy.descending("__time")))
               .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
               .dataSource("some src")
               .intervals(intervalSpec)
@@ -350,7 +358,7 @@ public class ScanQueryTest
     final ScanQuery scanQuery =
         Druids.newScanQueryBuilder()
               .columns("quality")
-              .orderBy(Collections.singletonList(new ScanQuery.OrderBy("quality", ScanQuery.Order.ASCENDING)))
+              .orderBy(Collections.singletonList(OrderBy.ascending("quality")))
               .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
               .dataSource("some src")
               .intervals(intervalSpec)
@@ -364,7 +372,7 @@ public class ScanQueryTest
   {
     final ScanQuery query =
         Druids.newScanQueryBuilder()
-              .order(ScanQuery.Order.DESCENDING)
+              .order(Order.DESCENDING)
               .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
               .dataSource("some src")
               .intervals(intervalSpec)
@@ -378,7 +386,7 @@ public class ScanQueryTest
   {
     final ScanQuery query =
         Druids.newScanQueryBuilder()
-              .order(ScanQuery.Order.DESCENDING)
+              .order(Order.DESCENDING)
               .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
               .dataSource("some src")
               .intervals(intervalSpec)
@@ -419,4 +427,26 @@ public class ScanQueryTest
 
     Assert.assertEquals(sig, query.getRowSignature());
   }
+
+  @Test
+  public void testAsCursorBuildSpec()
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        new ExpressionVirtualColumn("v0", "concat(placement, 'foo')", ColumnType.STRING, ExprMacroTable.nil())
+    );
+    final ScanQuery query =
+        Druids.newScanQueryBuilder()
+              .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
+              .dataSource("some src")
+              .intervals(QueryRunnerTestHelper.FIRST_TO_THIRD)
+              .virtualColumns(virtualColumns)
+              .columns("foo", "bar")
+              .build();
+
+    final CursorBuildSpec buildSpec = ScanQueryEngine.makeCursorBuildSpec(query, null);
+    Assert.assertEquals(QueryRunnerTestHelper.FIRST_TO_THIRD.getIntervals().get(0), buildSpec.getInterval());
+    Assert.assertNull(buildSpec.getGroupingColumns());
+    Assert.assertNull(buildSpec.getAggregators());
+    Assert.assertEquals(virtualColumns, buildSpec.getVirtualColumns());
+  }
 }
diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanResultValueTimestampComparatorTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanResultValueTimestampComparatorTest.java
index 47cc4208640..91f0ef0c4a6 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/ScanResultValueTimestampComparatorTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/ScanResultValueTimestampComparatorTest.java
@@ -21,6 +21,7 @@ package org.apache.druid.query.scan;
 
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
 import org.apache.druid.query.spec.QuerySegmentSpec;
 import org.apache.druid.segment.column.ColumnHolder;
@@ -52,7 +53,7 @@ public class ScanResultValueTimestampComparatorTest
   public void testComparisonDescendingList()
   {
     ScanQuery query = Druids.newScanQueryBuilder()
-                            .order(ScanQuery.Order.DESCENDING)
+                            .order(Order.DESCENDING)
                             .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
                             .dataSource("some src")
                             .intervals(intervalSpec)
@@ -89,7 +90,7 @@ public class ScanResultValueTimestampComparatorTest
   public void testComparisonAscendingList()
   {
     ScanQuery query = Druids.newScanQueryBuilder()
-                            .order(ScanQuery.Order.ASCENDING)
+                            .order(Order.ASCENDING)
                             .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_LIST)
                             .dataSource("some src")
                             .intervals(intervalSpec)
@@ -126,7 +127,7 @@ public class ScanResultValueTimestampComparatorTest
   public void testComparisonDescendingCompactedList()
   {
     ScanQuery query = Druids.newScanQueryBuilder()
-                            .order(ScanQuery.Order.DESCENDING)
+                            .order(Order.DESCENDING)
                             .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                             .dataSource("some src")
                             .intervals(intervalSpec)
@@ -161,7 +162,7 @@ public class ScanResultValueTimestampComparatorTest
   public void testAscendingCompactedList()
   {
     ScanQuery query = Druids.newScanQueryBuilder()
-                            .order(ScanQuery.Order.ASCENDING)
+                            .order(Order.ASCENDING)
                             .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                             .dataSource("some src")
                             .intervals(intervalSpec)
diff --git a/processing/src/test/java/org/apache/druid/query/scan/UnnestScanQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/scan/UnnestScanQueryRunnerTest.java
index 23d7f7a96af..a69e42bcd4d 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/UnnestScanQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/UnnestScanQueryRunnerTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Lists;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.query.DefaultGenericQueryMetricsFactory;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryRunner;
 import org.apache.druid.query.QueryRunnerTestHelper;
@@ -308,7 +309,7 @@ public class UnnestScanQueryRunnerTest extends InitializedNullHandlingTest
         .columns(QueryRunnerTestHelper.TIME_DIMENSION, QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST)
         .limit(3)
         .filters(new SelectorDimFilter(QueryRunnerTestHelper.MARKET_DIMENSION, "spot", null))
-        .order(ScanQuery.Order.ASCENDING)
+        .order(Order.ASCENDING)
         .build();
 
 
diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
index f555d81cefe..727086b7be3 100644
--- a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
@@ -37,10 +37,11 @@ import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.query.filter.AndDimFilter;
 import org.apache.druid.query.filter.EqualityFilter;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.FilterTuning;
 import org.apache.druid.query.filter.NullFilter;
 import org.apache.druid.query.filter.OrDimFilter;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.column.ColumnType;
@@ -661,18 +662,17 @@ public class NestedDataTimeseriesQueryTest extends InitializedNullHandlingTest
     List<Segment> segments = segmentsGenerator.apply(tempFolder, closer);
     Supplier<List<Result<TimeseriesResultValue>>> runner =
         () -> helper.runQueryOnSegmentsObjs(segments, timeseriesQuery).toList();
-    Filter filter = timeseriesQuery.getFilter() == null ? null : timeseriesQuery.getFilter().toFilter();
+    final CursorBuildSpec spec = TimeseriesQueryEngine.makeCursorBuildSpec(timeseriesQuery, null);
     boolean allCanVectorize = segments.stream()
                                       .allMatch(
-                                          s -> s.asStorageAdapter()
-                                                .canVectorize(
-                                                    filter,
-                                                    timeseriesQuery.getVirtualColumns(),
-                                                    timeseriesQuery.isDescending()
-                                                )
-                                      );
+                                          s -> {
+                                            final CursorHolder cursorHolder = s.asStorageAdapter()
+                                                                               .makeCursorHolder(spec);
+                                            final boolean canVectorize = cursorHolder.canVectorize();
+                                            cursorHolder.close();
+                                            return canVectorize;
+                                          });
 
-    Assert.assertEquals(NestedDataTestUtils.expectSegmentGeneratorCanVectorize(segmentsName), allCanVectorize);
     if (!allCanVectorize) {
       if (vectorize == QueryContexts.Vectorize.FORCE) {
         Throwable t = Assert.assertThrows(RuntimeException.class, runner::get);
diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryTest.java
index e8f10da8bf3..c1f8a144276 100644
--- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryTest.java
@@ -20,15 +20,20 @@
 package org.apache.druid.query.timeseries;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryRunnerTestHelper;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
+import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -36,9 +41,10 @@ import org.junit.runners.Parameterized;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 
 @RunWith(Parameterized.class)
-public class TimeseriesQueryTest
+public class TimeseriesQueryTest extends InitializedNullHandlingTest
 {
   private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper();
 
@@ -101,4 +107,99 @@ public class TimeseriesQueryTest
 
     Assert.assertEquals(ImmutableSet.of("__time", "fieldFromVirtualColumn", "aField"), query.getRequiredColumns());
   }
+
+  @Test
+  public void testAsCursorBuildSpecAllGranularity()
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        new ExpressionVirtualColumn(
+            "index",
+            "\"fieldFromVirtualColumn\"",
+            ColumnType.LONG,
+            ExprMacroTable.nil()
+        )
+    );
+    final LongSumAggregatorFactory beep = new LongSumAggregatorFactory("beep", "aField");
+    final TimeseriesQuery query =
+        Druids.newTimeseriesQueryBuilder()
+              .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
+              .granularity(Granularities.ALL)
+              .virtualColumns(virtualColumns)
+              .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
+              .aggregators(
+                  QueryRunnerTestHelper.ROWS_COUNT,
+                  QueryRunnerTestHelper.INDEX_DOUBLE_SUM,
+                  QueryRunnerTestHelper.INDEX_LONG_MAX,
+                  beep
+              )
+              .postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT)
+              .descending(descending)
+              .build();
+
+    final CursorBuildSpec buildSpec = TimeseriesQueryEngine.makeCursorBuildSpec(query, null);
+    Assert.assertEquals(QueryRunnerTestHelper.FULL_ON_INTERVAL, buildSpec.getInterval());
+    Assert.assertNull(buildSpec.getGroupingColumns());
+    Assert.assertEquals(
+        ImmutableList.of(
+            QueryRunnerTestHelper.ROWS_COUNT,
+            QueryRunnerTestHelper.INDEX_DOUBLE_SUM,
+            QueryRunnerTestHelper.INDEX_LONG_MAX,
+            beep
+        ),
+        buildSpec.getAggregators()
+    );
+    Assert.assertEquals(virtualColumns, buildSpec.getVirtualColumns());
+  }
+
+  @Test
+  public void testAsCursorBuildSpecDayGranularity()
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        new ExpressionVirtualColumn(
+            "index",
+            "\"fieldFromVirtualColumn\"",
+            ColumnType.LONG,
+            ExprMacroTable.nil()
+        )
+    );
+    final LongSumAggregatorFactory beep = new LongSumAggregatorFactory("beep", "aField");
+    final TimeseriesQuery query =
+        Druids.newTimeseriesQueryBuilder()
+              .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
+              .granularity(Granularities.DAY)
+              .virtualColumns(virtualColumns)
+              .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
+              .aggregators(
+                  QueryRunnerTestHelper.ROWS_COUNT,
+                  QueryRunnerTestHelper.INDEX_DOUBLE_SUM,
+                  QueryRunnerTestHelper.INDEX_LONG_MAX,
+                  beep
+              )
+              .postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT)
+              .descending(descending)
+              .build();
+
+    final CursorBuildSpec buildSpec = TimeseriesQueryEngine.makeCursorBuildSpec(query, null);
+    Assert.assertEquals(QueryRunnerTestHelper.FULL_ON_INTERVAL, buildSpec.getInterval());
+    Assert.assertEquals(
+        Collections.singletonList(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME),
+        buildSpec.getGroupingColumns()
+    );
+    Assert.assertEquals(
+        ImmutableList.of(
+            QueryRunnerTestHelper.ROWS_COUNT,
+            QueryRunnerTestHelper.INDEX_DOUBLE_SUM,
+            QueryRunnerTestHelper.INDEX_LONG_MAX,
+            beep
+        ),
+        buildSpec.getAggregators()
+    );
+    Assert.assertEquals(
+        VirtualColumns.create(
+            Granularities.toVirtualColumn(query.getGranularity(), Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME),
+            virtualColumns.getVirtualColumns()[0]
+        ),
+        buildSpec.getVirtualColumns()
+    );
+  }
 }
diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNMetricSpecOptimizationsTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNMetricSpecOptimizationsTest.java
index f68104ff481..5c5cba6ece6 100644
--- a/processing/src/test/java/org/apache/druid/query/topn/TopNMetricSpecOptimizationsTest.java
+++ b/processing/src/test/java/org/apache/druid/query/topn/TopNMetricSpecOptimizationsTest.java
@@ -24,23 +24,20 @@ import com.google.common.collect.Lists;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.QueryRunnerTestHelper;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory;
 import org.apache.druid.query.aggregation.DoubleMinAggregatorFactory;
 import org.apache.druid.query.filter.DruidPredicateFactory;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IdLookup;
 import org.apache.druid.segment.Metadata;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.IndexedInts;
@@ -321,16 +318,17 @@ public class TopNMetricSpecOptimizationsTest
       }
 
       @Override
-      public Sequence<Cursor> makeCursors(
-          @Nullable Filter filter,
-          Interval interval,
-          VirtualColumns virtualColumns,
-          Granularity gran,
-          boolean descending,
-          @Nullable QueryMetrics<?> queryMetrics
-      )
+      public CursorHolder makeCursorHolder(CursorBuildSpec spec)
       {
-        return null;
+        return new CursorHolder()
+        {
+          @Nullable
+          @Override
+          public Cursor asCursor()
+          {
+            return null;
+          }
+        };
       }
     };
 
diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java
index 79adf89fb86..b0f015c627f 100644
--- a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java
@@ -6193,6 +6193,165 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
     assertExpectedResults(expectedResults, query);
   }
 
+  @Test
+  public void testTopNAggregateTopnMetricFirst()
+  {
+    TopNQuery query = new TopNQueryBuilder()
+            .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
+            .granularity(QueryRunnerTestHelper.ALL_GRAN)
+            .dimension(QueryRunnerTestHelper.MARKET_DIMENSION)
+            .metric(QueryRunnerTestHelper.INDEX_METRIC)
+            .threshold(4)
+            .intervals(QueryRunnerTestHelper.FIRST_TO_THIRD)
+            .aggregators(commonAggregators)
+            .postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT)
+            .context(ImmutableMap.of("doAggregateTopNMetricFirst", true))
+            .build();
+
+
+    List<Result<TopNResultValue>> expectedResults = Collections.singletonList(
+            new Result<>(
+                    DateTimes.of("2011-04-01T00:00:00.000Z"),
+                    TopNResultValue.create(
+                            Arrays.<Map<String, Object>>asList(
+                                    ImmutableMap.of(
+                                            QueryRunnerTestHelper.MARKET_DIMENSION, "total_market",
+                                            "rows", 4L,
+                                            "index", 5351.814783D,
+                                            "addRowsIndexConstant", 5356.814783D,
+                                            "uniques", QueryRunnerTestHelper.UNIQUES_2
+                                    ),
+                                    ImmutableMap.of(
+                                            QueryRunnerTestHelper.MARKET_DIMENSION, "upfront",
+                                            "rows", 4L,
+                                            "index", 4875.669692D,
+                                            "addRowsIndexConstant", 4880.669692D,
+                                            "uniques", QueryRunnerTestHelper.UNIQUES_2
+                                    ),
+                                    ImmutableMap.of(
+                                            QueryRunnerTestHelper.MARKET_DIMENSION, "spot",
+                                            "rows", 18L,
+                                            "index", 2231.876812D,
+                                            "addRowsIndexConstant", 2250.876812D,
+                                            "uniques", QueryRunnerTestHelper.UNIQUES_9
+                                    )
+                            )
+                    )
+            )
+    );
+    assertExpectedResults(expectedResults, query);
+  }
+
+  @Test
+  public void testTopNAggregateTopnMetricFirstWithGranularity()
+  {
+    TopNQuery query = new TopNQueryBuilder()
+        .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
+        .granularity(QueryRunnerTestHelper.MONTH_GRAN)
+        .dimension(QueryRunnerTestHelper.MARKET_DIMENSION)
+        .metric("last")
+        .threshold(3)
+        .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
+        .aggregators(
+            new LongFirstAggregatorFactory("first", "index", null),
+            new LongLastAggregatorFactory("last", "index", null)
+        )
+        .context(ImmutableMap.of("doAggregateTopNMetricFirst", true))
+        .build();
+
+    List<Result<TopNResultValue>> expectedResults = Arrays.asList(
+        new Result<>(
+            DateTimes.of("2011-01-01T00:00:00.000Z"),
+            TopNResultValue.create(
+                Arrays.<Map<String, Object>>asList(
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "total_market")
+                                .put("first", 1000L)
+                                .put("last", 1127L)
+                                .build(),
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "upfront")
+                                .put("first", 800L)
+                                .put("last", 943L)
+                                .build(),
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "spot")
+                                .put("first", 100L)
+                                .put("last", 155L)
+                                .build()
+                )
+            )
+        ),
+        new Result<>(
+            DateTimes.of("2011-02-01T00:00:00.000Z"),
+            TopNResultValue.create(
+                Arrays.<Map<String, Object>>asList(
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "total_market")
+                                .put("first", 1203L)
+                                .put("last", 1292L)
+                                .build(),
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "upfront")
+                                .put("first", 1667L)
+                                .put("last", 1101L)
+                                .build(),
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "spot")
+                                .put("first", 132L)
+                                .put("last", 114L)
+                                .build()
+                )
+            )
+        ),
+        new Result<>(
+            DateTimes.of("2011-03-01T00:00:00.000Z"),
+            TopNResultValue.create(
+                Arrays.<Map<String, Object>>asList(
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "total_market")
+                                .put("first", 1124L)
+                                .put("last", 1366L)
+                                .build(),
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "upfront")
+                                .put("first", 1166L)
+                                .put("last", 1063L)
+                                .build(),
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "spot")
+                                .put("first", 153L)
+                                .put("last", 125L)
+                                .build()
+                )
+            )
+        ),
+        new Result<>(
+            DateTimes.of("2011-04-01T00:00:00.000Z"),
+            TopNResultValue.create(
+                Arrays.<Map<String, Object>>asList(
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "total_market")
+                                .put("first", 1314L)
+                                .put("last", 1029L)
+                                .build(),
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "upfront")
+                                .put("first", 1447L)
+                                .put("last", 780L)
+                                .build(),
+                    ImmutableMap.<String, Object>builder()
+                                .put("market", "spot")
+                                .put("first", 135L)
+                                .put("last", 120L)
+                                .build()
+                )
+            )
+        )
+    );
+    assertExpectedResults(expectedResults, query);
+  }
+
   private static Map<String, Object> makeRowWithNulls(
       String dimName,
       @Nullable Object dimValue,
diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryTest.java
index 6d7f1a03be9..1fd483eabe0 100644
--- a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryTest.java
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryRunnerTestHelper;
@@ -38,9 +39,12 @@ import org.apache.druid.query.dimension.LegacyDimensionSpec;
 import org.apache.druid.query.extraction.MapLookupExtractor;
 import org.apache.druid.query.lookup.LookupExtractionFn;
 import org.apache.druid.query.ordering.StringComparators;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
+import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -48,7 +52,7 @@ import org.junit.rules.ExpectedException;
 
 import java.io.IOException;
 
-public class TopNQueryTest
+public class TopNQueryTest extends InitializedNullHandlingTest
 {
   @Rule
   public ExpectedException expectedException = ExpectedException.none();
@@ -266,4 +270,65 @@ public class TopNQueryTest
 
     Assert.assertEquals(ImmutableSet.of("__time", "other", "index"), query.getRequiredColumns());
   }
+
+  @Test
+  public void testAsCursorBuildSpecAllGranularity()
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        new ExpressionVirtualColumn("v", "\"other\"", ColumnType.STRING, ExprMacroTable.nil())
+    );
+    final LongSumAggregatorFactory longSum = new LongSumAggregatorFactory("idx", "index");
+    final TopNQuery query = new TopNQueryBuilder()
+        .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
+        .intervals(QueryRunnerTestHelper.FIRST_TO_THIRD)
+        .virtualColumns(virtualColumns)
+        .dimension(DefaultDimensionSpec.of("v"))
+        .aggregators(QueryRunnerTestHelper.ROWS_COUNT, longSum)
+        .granularity(Granularities.ALL)
+        .postAggregators(ImmutableList.of(new FieldAccessPostAggregator("x", "idx")))
+        .metric(new NumericTopNMetricSpec("idx"))
+        .threshold(100)
+        .build();
+
+    final CursorBuildSpec buildSpec = TopNQueryEngine.makeCursorBuildSpec(query, null);
+    Assert.assertEquals(QueryRunnerTestHelper.FIRST_TO_THIRD.getIntervals().get(0), buildSpec.getInterval());
+    Assert.assertEquals(ImmutableList.of("v"), buildSpec.getGroupingColumns());
+    Assert.assertEquals(ImmutableList.of(QueryRunnerTestHelper.ROWS_COUNT, longSum), buildSpec.getAggregators());
+    Assert.assertEquals(virtualColumns, buildSpec.getVirtualColumns());
+  }
+
+  @Test
+  public void testAsCursorBuildSpecDayGranularity()
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        new ExpressionVirtualColumn("v", "\"other\"", ColumnType.STRING, ExprMacroTable.nil())
+    );
+    final LongSumAggregatorFactory longSum = new LongSumAggregatorFactory("idx", "index");
+    final TopNQuery query = new TopNQueryBuilder()
+        .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
+        .intervals(QueryRunnerTestHelper.FIRST_TO_THIRD)
+        .virtualColumns(virtualColumns)
+        .dimension(DefaultDimensionSpec.of("v"))
+        .aggregators(QueryRunnerTestHelper.ROWS_COUNT, longSum)
+        .granularity(Granularities.DAY)
+        .postAggregators(ImmutableList.of(new FieldAccessPostAggregator("x", "idx")))
+        .metric(new NumericTopNMetricSpec("idx"))
+        .threshold(100)
+        .build();
+
+    final CursorBuildSpec buildSpec = TopNQueryEngine.makeCursorBuildSpec(query, null);
+    Assert.assertEquals(QueryRunnerTestHelper.FIRST_TO_THIRD.getIntervals().get(0), buildSpec.getInterval());
+    Assert.assertEquals(
+        ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "v"),
+        buildSpec.getGroupingColumns()
+    );
+    Assert.assertEquals(ImmutableList.of(QueryRunnerTestHelper.ROWS_COUNT, longSum), buildSpec.getAggregators());
+    Assert.assertEquals(
+        VirtualColumns.create(
+            Granularities.toVirtualColumn(query.getGranularity(), Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME),
+            virtualColumns.getVirtualColumns()[0]
+        ),
+        buildSpec.getVirtualColumns()
+    );
+  }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java b/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java
index 73a2b6db50a..788181859ff 100644
--- a/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java
@@ -26,9 +26,7 @@ import org.apache.druid.data.input.MapBasedInputRow;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.guice.BuiltInTypesModule;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.DimensionSpec;
@@ -46,7 +44,6 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import javax.annotation.Nonnull;
-import java.util.List;
 import java.util.Map;
 
 public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest
@@ -151,58 +148,44 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, STRING_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(STRING_COL, STRING_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(STRING_COL, STRING_COL, ColumnType.STRING);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("a", valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("a", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("a", dimensionSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
+      DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals("a", valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("a", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("a", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("b", valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("b", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("b", dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals("b", valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("b", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("b", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("c", valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("c", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("c", dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals("c", valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("c", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("c", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertNull(dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertNull(dimensionSelector.getObject());
 
-    Assert.assertEquals(ColumnType.STRING, storageAdapter.getColumnCapabilities(STRING_COL).toColumnType());
+      Assert.assertEquals(ColumnType.STRING, storageAdapter.getColumnCapabilities(STRING_COL).toColumnType());
+    }
   }
 
   @Test
@@ -218,88 +201,74 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, LONG_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(LONG_COL, LONG_COL, ColumnType.LONG);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(LONG_COL, LONG_COL, ColumnType.LONG);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(1L, valueSelector.getObject());
-    Assert.assertEquals(1L, valueSelector.getLong());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("1", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(2L, valueSelector.getObject());
-    Assert.assertEquals(2L, valueSelector.getLong());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("2", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("2", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(3L, valueSelector.getObject());
-    Assert.assertEquals(3L, valueSelector.getLong());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("3", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("3", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    if (NullHandling.sqlCompatible()) {
-      Assert.assertNull(valueSelector.getObject());
-      Assert.assertTrue(valueSelector.isNull());
-      Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-      Assert.assertNull(dimensionSelector.getObject());
-    } else {
-      Assert.assertEquals(NullHandling.defaultLongValue(), valueSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
+      DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals(1L, valueSelector.getObject());
+      Assert.assertEquals(1L, valueSelector.getLong());
       Assert.assertFalse(valueSelector.isNull());
       Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertEquals(
-          String.valueOf(NullHandling.defaultLongValue()),
-          dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
-      );
-      Assert.assertEquals(String.valueOf(NullHandling.defaultLongValue()), dimensionSelector.getObject());
-    }
+      Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("1", dimensionSelector.getObject());
 
-
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    if (NullHandling.sqlCompatible()) {
-      Assert.assertNull(valueSelector.getObject());
-      Assert.assertTrue(valueSelector.isNull());
-      Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-      Assert.assertNull(dimensionSelector.getObject());
-    } else {
-      Assert.assertEquals(NullHandling.defaultLongValue(), valueSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(2L, valueSelector.getObject());
+      Assert.assertEquals(2L, valueSelector.getLong());
       Assert.assertFalse(valueSelector.isNull());
       Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertEquals(
-          String.valueOf(NullHandling.defaultLongValue()),
-          dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
-      );
-      Assert.assertEquals(String.valueOf(NullHandling.defaultLongValue()), dimensionSelector.getObject());
+      Assert.assertEquals("2", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("2", dimensionSelector.getObject());
+
+      cursor.advance();
+      Assert.assertEquals(3L, valueSelector.getObject());
+      Assert.assertEquals(3L, valueSelector.getLong());
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("3", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("3", dimensionSelector.getObject());
+
+      cursor.advance();
+      if (NullHandling.sqlCompatible()) {
+        Assert.assertNull(valueSelector.getObject());
+        Assert.assertTrue(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+        Assert.assertNull(dimensionSelector.getObject());
+      } else {
+        Assert.assertEquals(NullHandling.defaultLongValue(), valueSelector.getObject());
+        Assert.assertFalse(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertEquals(
+            String.valueOf(NullHandling.defaultLongValue()),
+            dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
+        );
+        Assert.assertEquals(String.valueOf(NullHandling.defaultLongValue()), dimensionSelector.getObject());
+      }
+
+
+      cursor.advance();
+      if (NullHandling.sqlCompatible()) {
+        Assert.assertNull(valueSelector.getObject());
+        Assert.assertTrue(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+        Assert.assertNull(dimensionSelector.getObject());
+      } else {
+        Assert.assertEquals(NullHandling.defaultLongValue(), valueSelector.getObject());
+        Assert.assertFalse(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertEquals(
+            String.valueOf(NullHandling.defaultLongValue()),
+            dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
+        );
+        Assert.assertEquals(String.valueOf(NullHandling.defaultLongValue()), dimensionSelector.getObject());
+      }
+      Assert.assertEquals(ColumnType.LONG, storageAdapter.getColumnCapabilities(LONG_COL).toColumnType());
     }
-    Assert.assertEquals(ColumnType.LONG, storageAdapter.getColumnCapabilities(LONG_COL).toColumnType());
   }
 
   @Test
@@ -315,87 +284,74 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, DOUBLE_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(DOUBLE_COL, DOUBLE_COL, ColumnType.DOUBLE);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(DOUBLE_COL, DOUBLE_COL, ColumnType.DOUBLE);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(1.1, valueSelector.getObject());
-    Assert.assertEquals(1.1, valueSelector.getDouble(), 0.0);
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("1.1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("1.1", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(2.2, valueSelector.getObject());
-    Assert.assertEquals(2.2, valueSelector.getDouble(), 0.0);
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("2.2", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("2.2", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(3.3, valueSelector.getObject());
-    Assert.assertEquals(3.3, valueSelector.getDouble(), 0.0);
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("3.3", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("3.3", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    if (NullHandling.sqlCompatible()) {
-      Assert.assertNull(valueSelector.getObject());
-      Assert.assertTrue(valueSelector.isNull());
-      Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-      Assert.assertNull(dimensionSelector.getObject());
-    } else {
-      Assert.assertEquals(NullHandling.defaultDoubleValue(), valueSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
+      DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals(1.1, valueSelector.getObject());
+      Assert.assertEquals(1.1, valueSelector.getDouble(), 0.0);
       Assert.assertFalse(valueSelector.isNull());
       Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertEquals(
-          String.valueOf(NullHandling.defaultDoubleValue()),
-          dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
-      );
-      Assert.assertEquals(String.valueOf(NullHandling.defaultDoubleValue()), dimensionSelector.getObject());
-    }
+      Assert.assertEquals("1.1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("1.1", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    if (NullHandling.sqlCompatible()) {
-      Assert.assertNull(valueSelector.getObject());
-      Assert.assertTrue(valueSelector.isNull());
-      Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-      Assert.assertNull(dimensionSelector.getObject());
-    } else {
-      Assert.assertEquals(NullHandling.defaultDoubleValue(), valueSelector.getObject());
+
+      cursor.advance();
+      Assert.assertEquals(2.2, valueSelector.getObject());
+      Assert.assertEquals(2.2, valueSelector.getDouble(), 0.0);
       Assert.assertFalse(valueSelector.isNull());
       Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertEquals(
-          String.valueOf(NullHandling.defaultDoubleValue()),
-          dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
-      );
-      Assert.assertEquals(String.valueOf(NullHandling.defaultDoubleValue()), dimensionSelector.getObject());
+      Assert.assertEquals("2.2", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("2.2", dimensionSelector.getObject());
+
+      cursor.advance();
+      Assert.assertEquals(3.3, valueSelector.getObject());
+      Assert.assertEquals(3.3, valueSelector.getDouble(), 0.0);
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("3.3", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("3.3", dimensionSelector.getObject());
+
+      cursor.advance();
+      if (NullHandling.sqlCompatible()) {
+        Assert.assertNull(valueSelector.getObject());
+        Assert.assertTrue(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+        Assert.assertNull(dimensionSelector.getObject());
+      } else {
+        Assert.assertEquals(NullHandling.defaultDoubleValue(), valueSelector.getObject());
+        Assert.assertFalse(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertEquals(
+            String.valueOf(NullHandling.defaultDoubleValue()),
+            dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
+        );
+        Assert.assertEquals(String.valueOf(NullHandling.defaultDoubleValue()), dimensionSelector.getObject());
+      }
+
+      cursor.advance();
+      if (NullHandling.sqlCompatible()) {
+        Assert.assertNull(valueSelector.getObject());
+        Assert.assertTrue(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+        Assert.assertNull(dimensionSelector.getObject());
+      } else {
+        Assert.assertEquals(NullHandling.defaultDoubleValue(), valueSelector.getObject());
+        Assert.assertFalse(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertEquals(
+            String.valueOf(NullHandling.defaultDoubleValue()),
+            dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
+        );
+        Assert.assertEquals(String.valueOf(NullHandling.defaultDoubleValue()), dimensionSelector.getObject());
+      }
+      Assert.assertEquals(ColumnType.DOUBLE, storageAdapter.getColumnCapabilities(DOUBLE_COL).toColumnType());
     }
-    Assert.assertEquals(ColumnType.DOUBLE, storageAdapter.getColumnCapabilities(DOUBLE_COL).toColumnType());
   }
 
   @Test
@@ -411,57 +367,39 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, STRING_ARRAY_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(STRING_ARRAY_COL, STRING_ARRAY_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(
+          STRING_ARRAY_COL,
+          STRING_ARRAY_COL,
+          ColumnType.STRING
+      );
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(0).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertArrayEquals(new Object[]{"a"}, (Object[]) valueSelector.getObject());
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(1).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertArrayEquals(new Object[]{"b", "c"}, (Object[]) valueSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
+      Assert.assertThrows(
+          UnsupportedOperationException.class,
+          () -> cursor.getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
+      );
+      Assert.assertArrayEquals(new Object[]{"a"}, (Object[]) valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(2).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertArrayEquals(new Object[]{"d", "e"}, (Object[]) valueSelector.getObject());
+      cursor.advance();
+      Assert.assertArrayEquals(new Object[]{"b", "c"}, (Object[]) valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(3).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertNull(valueSelector.getObject());
+      cursor.advance();
+      Assert.assertArrayEquals(new Object[]{"d", "e"}, (Object[]) valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(4).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertEquals(ColumnType.STRING_ARRAY, storageAdapter.getColumnCapabilities(STRING_ARRAY_COL).toColumnType());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertEquals(
+          ColumnType.STRING_ARRAY,
+          storageAdapter.getColumnCapabilities(STRING_ARRAY_COL).toColumnType()
+      );
+    }
   }
 
   @Test
@@ -477,50 +415,35 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, VARIANT_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(VARIANT_COL, VARIANT_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(VARIANT_COL, VARIANT_COL, ColumnType.STRING);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    DimensionSelector dimensionSelector = cursorList.get(0).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("a", valueSelector.getObject());
-    Assert.assertEquals("a", dimensionSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
+      DimensionSelector dimensionSelector = cursor.getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals("a", valueSelector.getObject());
+      Assert.assertEquals("a", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    dimensionSelector = cursorList.get(1).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(2L, valueSelector.getObject());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals("2", dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(2L, valueSelector.getObject());
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals("2", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    dimensionSelector = cursorList.get(2).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(3.3, valueSelector.getObject());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals("3.3", dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(3.3, valueSelector.getObject());
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals("3.3", dimensionSelector.getObject());
 
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertNull(dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    dimensionSelector = cursorList.get(3).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertNull(dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    dimensionSelector = cursorList.get(4).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertNull(dimensionSelector.getObject());
-    Assert.assertEquals(ColumnType.STRING, storageAdapter.getColumnCapabilities(VARIANT_COL).toColumnType());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertNull(dimensionSelector.getObject());
+      Assert.assertEquals(ColumnType.STRING, storageAdapter.getColumnCapabilities(VARIANT_COL).toColumnType());
+    }
   }
 
   @Test
@@ -536,57 +459,31 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, NESTED_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(NESTED_COL, NESTED_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(NESTED_COL, NESTED_COL, ColumnType.STRING);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(0).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertEquals(StructuredData.wrap("a"), valueSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
+      Assert.assertThrows(
+          UnsupportedOperationException.class,
+          () -> cursor.getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
+      );
+      Assert.assertEquals(StructuredData.wrap("a"), valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(1).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertEquals(StructuredData.wrap(2L), valueSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(StructuredData.wrap(2L), valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(2).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertEquals(StructuredData.wrap(ImmutableMap.of("x", 1.1, "y", 2L)), valueSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(StructuredData.wrap(ImmutableMap.of("x", 1.1, "y", 2L)), valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(3).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertNull(valueSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(4).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertEquals(ColumnType.NESTED_DATA, storageAdapter.getColumnCapabilities(NESTED_COL).toColumnType());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertEquals(ColumnType.NESTED_DATA, storageAdapter.getColumnCapabilities(NESTED_COL).toColumnType());
+    }
   }
 
   @Test
@@ -621,49 +518,35 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, NESTED_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(NESTED_COL, NESTED_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(NESTED_COL, NESTED_COL, ColumnType.STRING);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    DimensionSelector dimensionSelector = cursorList.get(0).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("a", valueSelector.getObject());
-    Assert.assertEquals("a", dimensionSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
+      DimensionSelector dimensionSelector = cursor.getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals("a", valueSelector.getObject());
+      Assert.assertEquals("a", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    dimensionSelector = cursorList.get(1).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("2", valueSelector.getObject());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals("2", dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals("2", valueSelector.getObject());
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals("2", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    dimensionSelector = cursorList.get(2).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertNull(dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    dimensionSelector = cursorList.get(3).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertNull(dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    dimensionSelector = cursorList.get(4).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertNull(dimensionSelector.getObject());
 
-    Assert.assertEquals(ColumnType.STRING, storageAdapter.getColumnCapabilities(NESTED_COL).toColumnType());
+      Assert.assertEquals(ColumnType.STRING, storageAdapter.getColumnCapabilities(NESTED_COL).toColumnType());
+    }
   }
 
   @Test
diff --git a/processing/src/test/java/org/apache/druid/segment/ListCursor.java b/processing/src/test/java/org/apache/druid/segment/ListCursor.java
index fc1eba743d4..2ed9e843150 100644
--- a/processing/src/test/java/org/apache/druid/segment/ListCursor.java
+++ b/processing/src/test/java/org/apache/druid/segment/ListCursor.java
@@ -26,7 +26,6 @@ import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.data.IndexedInts;
-import org.joda.time.DateTime;
 
 import javax.annotation.Nullable;
 import java.util.List;
@@ -189,12 +188,6 @@ public class ListCursor implements Cursor
     };
   }
 
-  @Override
-  public DateTime getTime()
-  {
-    throw new UnsupportedOperationException();
-  }
-
   @Override
   public void advance()
   {
diff --git a/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java b/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java
index 359d2de58cb..f13b6c5632a 100644
--- a/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java
+++ b/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java
@@ -26,9 +26,7 @@ import org.apache.druid.data.input.MapBasedInputRow;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.guice.BuiltInTypesModule;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.DimensionSpec;
@@ -45,7 +43,6 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import javax.annotation.Nonnull;
-import java.util.List;
 import java.util.Map;
 
 public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest
@@ -150,56 +147,43 @@ public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, STRING_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(STRING_COL, STRING_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("a", valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("a", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("a", dimensionSelector.getObject());
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(STRING_COL, STRING_COL, ColumnType.STRING);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("b", valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("b", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("b", dimensionSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
+      DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals("a", valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("a", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("a", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("c", valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("c", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("c", dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals("b", valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("b", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("b", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals("c", valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("c", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("c", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertNull(dimensionSelector.getObject());
+
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertNull(dimensionSelector.getObject());
+    }
   }
 
   @Test
@@ -215,85 +199,71 @@ public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, LONG_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(LONG_COL, LONG_COL, ColumnType.LONG);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(LONG_COL, LONG_COL, ColumnType.LONG);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(1L, valueSelector.getObject());
-    Assert.assertEquals(1L, valueSelector.getLong());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("1", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(2L, valueSelector.getObject());
-    Assert.assertEquals(2L, valueSelector.getLong());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("2", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("2", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(3L, valueSelector.getObject());
-    Assert.assertEquals(3L, valueSelector.getLong());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("3", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("3", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    if (NullHandling.sqlCompatible()) {
-      Assert.assertNull(valueSelector.getObject());
-      Assert.assertTrue(valueSelector.isNull());
-      Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-      Assert.assertNull(dimensionSelector.getObject());
-    } else {
-      Assert.assertEquals(NullHandling.defaultLongValue(), valueSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
+      DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals(1L, valueSelector.getObject());
+      Assert.assertEquals(1L, valueSelector.getLong());
       Assert.assertFalse(valueSelector.isNull());
       Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertEquals(
-          String.valueOf(NullHandling.defaultLongValue()),
-          dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
-      );
-      Assert.assertEquals(String.valueOf(NullHandling.defaultLongValue()), dimensionSelector.getObject());
-    }
+      Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("1", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(LONG_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    if (NullHandling.sqlCompatible()) {
-      Assert.assertNull(valueSelector.getObject());
-      Assert.assertTrue(valueSelector.isNull());
-      Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-      Assert.assertNull(dimensionSelector.getObject());
-    } else {
-      Assert.assertEquals(NullHandling.defaultLongValue(), valueSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(2L, valueSelector.getObject());
+      Assert.assertEquals(2L, valueSelector.getLong());
       Assert.assertFalse(valueSelector.isNull());
       Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertEquals(
-          String.valueOf(NullHandling.defaultLongValue()),
-          dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
-      );
-      Assert.assertEquals(String.valueOf(NullHandling.defaultLongValue()), dimensionSelector.getObject());
+      Assert.assertEquals("2", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("2", dimensionSelector.getObject());
+
+      cursor.advance();
+      Assert.assertEquals(3L, valueSelector.getObject());
+      Assert.assertEquals(3L, valueSelector.getLong());
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("3", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("3", dimensionSelector.getObject());
+
+      cursor.advance();
+      if (NullHandling.sqlCompatible()) {
+        Assert.assertNull(valueSelector.getObject());
+        Assert.assertTrue(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+        Assert.assertNull(dimensionSelector.getObject());
+      } else {
+        Assert.assertEquals(NullHandling.defaultLongValue(), valueSelector.getObject());
+        Assert.assertFalse(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertEquals(
+            String.valueOf(NullHandling.defaultLongValue()),
+            dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
+        );
+        Assert.assertEquals(String.valueOf(NullHandling.defaultLongValue()), dimensionSelector.getObject());
+      }
+
+      cursor.advance();
+      if (NullHandling.sqlCompatible()) {
+        Assert.assertNull(valueSelector.getObject());
+        Assert.assertTrue(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+        Assert.assertNull(dimensionSelector.getObject());
+      } else {
+        Assert.assertEquals(NullHandling.defaultLongValue(), valueSelector.getObject());
+        Assert.assertFalse(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertEquals(
+            String.valueOf(NullHandling.defaultLongValue()),
+            dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
+        );
+        Assert.assertEquals(String.valueOf(NullHandling.defaultLongValue()), dimensionSelector.getObject());
+      }
     }
   }
 
@@ -310,85 +280,71 @@ public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, DOUBLE_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(DOUBLE_COL, DOUBLE_COL, ColumnType.DOUBLE);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(DOUBLE_COL, DOUBLE_COL, ColumnType.DOUBLE);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(1.1, valueSelector.getObject());
-    Assert.assertEquals(1.1, valueSelector.getDouble(), 0.0);
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("1.1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("1.1", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(2.2, valueSelector.getObject());
-    Assert.assertEquals(2.2, valueSelector.getDouble(), 0.0);
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("2.2", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("2.2", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(3.3, valueSelector.getObject());
-    Assert.assertEquals(3.3, valueSelector.getDouble(), 0.0);
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals(1, dimensionSelector.getRow().size());
-    Assert.assertEquals("3.3", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-    Assert.assertEquals("3.3", dimensionSelector.getObject());
-
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    if (NullHandling.sqlCompatible()) {
-      Assert.assertNull(valueSelector.getObject());
-      Assert.assertTrue(valueSelector.isNull());
-      Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-      Assert.assertNull(dimensionSelector.getObject());
-    } else {
-      Assert.assertEquals(NullHandling.defaultDoubleValue(), valueSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
+      DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals(1.1, valueSelector.getObject());
+      Assert.assertEquals(1.1, valueSelector.getDouble(), 0.0);
       Assert.assertFalse(valueSelector.isNull());
       Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertEquals(
-          String.valueOf(NullHandling.defaultDoubleValue()),
-          dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
-      );
-      Assert.assertEquals(String.valueOf(NullHandling.defaultDoubleValue()), dimensionSelector.getObject());
-    }
+      Assert.assertEquals("1.1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("1.1", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(DOUBLE_COL);
-    dimensionSelector = columnSelectorFactory.makeDimensionSelector(dimensionSpec);
-    if (NullHandling.sqlCompatible()) {
-      Assert.assertNull(valueSelector.getObject());
-      Assert.assertTrue(valueSelector.isNull());
-      Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
-      Assert.assertNull(dimensionSelector.getObject());
-    } else {
-      Assert.assertEquals(NullHandling.defaultDoubleValue(), valueSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(2.2, valueSelector.getObject());
+      Assert.assertEquals(2.2, valueSelector.getDouble(), 0.0);
       Assert.assertFalse(valueSelector.isNull());
       Assert.assertEquals(1, dimensionSelector.getRow().size());
-      Assert.assertEquals(
-          String.valueOf(NullHandling.defaultDoubleValue()),
-          dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
-      );
-      Assert.assertEquals(String.valueOf(NullHandling.defaultDoubleValue()), dimensionSelector.getObject());
+      Assert.assertEquals("2.2", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("2.2", dimensionSelector.getObject());
+
+      cursor.advance();
+      Assert.assertEquals(3.3, valueSelector.getObject());
+      Assert.assertEquals(3.3, valueSelector.getDouble(), 0.0);
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals(1, dimensionSelector.getRow().size());
+      Assert.assertEquals("3.3", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+      Assert.assertEquals("3.3", dimensionSelector.getObject());
+
+      cursor.advance();
+      if (NullHandling.sqlCompatible()) {
+        Assert.assertNull(valueSelector.getObject());
+        Assert.assertTrue(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+        Assert.assertNull(dimensionSelector.getObject());
+      } else {
+        Assert.assertEquals(NullHandling.defaultDoubleValue(), valueSelector.getObject());
+        Assert.assertFalse(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertEquals(
+            String.valueOf(NullHandling.defaultDoubleValue()),
+            dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
+        );
+        Assert.assertEquals(String.valueOf(NullHandling.defaultDoubleValue()), dimensionSelector.getObject());
+      }
+
+      cursor.advance();
+      if (NullHandling.sqlCompatible()) {
+        Assert.assertNull(valueSelector.getObject());
+        Assert.assertTrue(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertNull(dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+        Assert.assertNull(dimensionSelector.getObject());
+      } else {
+        Assert.assertEquals(NullHandling.defaultDoubleValue(), valueSelector.getObject());
+        Assert.assertFalse(valueSelector.isNull());
+        Assert.assertEquals(1, dimensionSelector.getRow().size());
+        Assert.assertEquals(
+            String.valueOf(NullHandling.defaultDoubleValue()),
+            dimensionSelector.lookupName(dimensionSelector.getRow().get(0))
+        );
+        Assert.assertEquals(String.valueOf(NullHandling.defaultDoubleValue()), dimensionSelector.getObject());
+      }
     }
   }
 
@@ -405,56 +361,34 @@ public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, STRING_ARRAY_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(STRING_ARRAY_COL, STRING_ARRAY_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(
+          STRING_ARRAY_COL,
+          STRING_ARRAY_COL,
+          ColumnType.STRING
+      );
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(0).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertArrayEquals(new Object[]{"a"}, (Object[]) valueSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
+      Assert.assertThrows(
+          UnsupportedOperationException.class,
+          () -> cursor.getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
+      );
+      Assert.assertArrayEquals(new Object[]{"a"}, (Object[]) valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(1).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertArrayEquals(new Object[]{"b", "c"}, (Object[]) valueSelector.getObject());
+      cursor.advance();
+      Assert.assertArrayEquals(new Object[]{"b", "c"}, (Object[]) valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(2).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertArrayEquals(new Object[]{"d", "e"}, (Object[]) valueSelector.getObject());
+      cursor.advance();
+      Assert.assertArrayEquals(new Object[]{"d", "e"}, (Object[]) valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(3).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertNull(valueSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(STRING_ARRAY_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(4).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertNull(valueSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+    }
   }
 
   @Test
@@ -470,48 +404,34 @@ public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, VARIANT_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(VARIANT_COL, VARIANT_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(VARIANT_COL, VARIANT_COL, ColumnType.STRING);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    DimensionSelector dimensionSelector = cursorList.get(0).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals("a", valueSelector.getObject());
-    Assert.assertEquals("a", dimensionSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
+      DimensionSelector dimensionSelector = cursor.getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
+      Assert.assertEquals("a", valueSelector.getObject());
+      Assert.assertEquals("a", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    dimensionSelector = cursorList.get(1).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(2L, valueSelector.getObject());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals("2", dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(2L, valueSelector.getObject());
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals("2", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    dimensionSelector = cursorList.get(2).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertEquals(3.3, valueSelector.getObject());
-    Assert.assertFalse(valueSelector.isNull());
-    Assert.assertEquals("3.3", dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(3.3, valueSelector.getObject());
+      Assert.assertFalse(valueSelector.isNull());
+      Assert.assertEquals("3.3", dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    dimensionSelector = cursorList.get(3).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertNull(dimensionSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(VARIANT_COL);
-    dimensionSelector = cursorList.get(4).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec);
-    Assert.assertNull(valueSelector.getObject());
-    Assert.assertNull(dimensionSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+      Assert.assertNull(dimensionSelector.getObject());
+    }
   }
 
   @Test
@@ -527,56 +447,30 @@ public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest
     index.add(makeInputRow(minTimestamp + 5, false, NESTED_COL, null));
 
     IncrementalIndexStorageAdapter storageAdapter = new IncrementalIndexStorageAdapter(index);
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.NONE,
-        false,
-        null
-    );
-    final DimensionSpec dimensionSpec = new DefaultDimensionSpec(NESTED_COL, NESTED_COL, ColumnType.STRING);
-    List<Cursor> cursorList = cursorSequence.toList();
-    ColumnSelectorFactory columnSelectorFactory = cursorList.get(0).getColumnSelectorFactory();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final DimensionSpec dimensionSpec = new DefaultDimensionSpec(NESTED_COL, NESTED_COL, ColumnType.STRING);
+      ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
-    ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(0).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertEquals(StructuredData.wrap("a"), valueSelector.getObject());
+      ColumnValueSelector valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
+      Assert.assertThrows(
+          UnsupportedOperationException.class,
+          () -> cursor.getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
+      );
+      Assert.assertEquals(StructuredData.wrap("a"), valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(1).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(1).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertEquals(StructuredData.wrap(2L), valueSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(StructuredData.wrap(2L), valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(2).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(2).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertEquals(StructuredData.wrap(ImmutableMap.of("x", 1.1, "y", 2L)), valueSelector.getObject());
+      cursor.advance();
+      Assert.assertEquals(StructuredData.wrap(ImmutableMap.of("x", 1.1, "y", 2L)), valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(3).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(3).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertNull(valueSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
 
-    columnSelectorFactory = cursorList.get(4).getColumnSelectorFactory();
-    valueSelector = columnSelectorFactory.makeColumnValueSelector(NESTED_COL);
-    Assert.assertThrows(
-        UnsupportedOperationException.class,
-        () -> cursorList.get(4).getColumnSelectorFactory().makeDimensionSelector(dimensionSpec)
-    );
-    Assert.assertNull(valueSelector.getObject());
+      cursor.advance();
+      Assert.assertNull(valueSelector.getObject());
+    }
   }
 
   @Nonnull
diff --git a/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilderTest.java b/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java
similarity index 72%
rename from processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilderTest.java
rename to processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java
index ea93e1b75c2..349236a9dcf 100644
--- a/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilderTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java
@@ -25,7 +25,7 @@ import org.apache.druid.segment.data.ReadableOffset;
 import org.junit.Assert;
 import org.junit.Test;
 
-public class QueryableIndexCursorSequenceBuilderTest
+public class QueryableIndexCursorHolderTest
 {
   @Test
   public void testTimeSearch()
@@ -66,52 +66,52 @@ public class QueryableIndexCursorSequenceBuilderTest
 
     Assert.assertEquals(
         0,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 0, 0, values.length)
+        QueryableIndexCursorHolder.timeSearch(column, 0, 0, values.length)
     );
 
     Assert.assertEquals(
         2,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 0, 2, values.length)
+        QueryableIndexCursorHolder.timeSearch(column, 0, 2, values.length)
     );
 
     Assert.assertEquals(
         0,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 0, 0, values.length / 2)
+        QueryableIndexCursorHolder.timeSearch(column, 0, 0, values.length / 2)
     );
 
     Assert.assertEquals(
         1,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 1, 0, values.length)
+        QueryableIndexCursorHolder.timeSearch(column, 1, 0, values.length)
     );
 
     Assert.assertEquals(
         2,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 1, 2, values.length)
+        QueryableIndexCursorHolder.timeSearch(column, 1, 2, values.length)
     );
 
     Assert.assertEquals(
         1,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 1, 0, values.length / 2)
+        QueryableIndexCursorHolder.timeSearch(column, 1, 0, values.length / 2)
     );
 
     Assert.assertEquals(
         1,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 1, 1, 8)
+        QueryableIndexCursorHolder.timeSearch(column, 1, 1, 8)
     );
 
     Assert.assertEquals(
         8,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 2, 0, values.length)
+        QueryableIndexCursorHolder.timeSearch(column, 2, 0, values.length)
     );
 
     Assert.assertEquals(
         10,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 10, 0, values.length)
+        QueryableIndexCursorHolder.timeSearch(column, 10, 0, values.length)
     );
 
     Assert.assertEquals(
         11,
-        QueryableIndexCursorSequenceBuilder.timeSearch(column, 15, 0, values.length)
+        QueryableIndexCursorHolder.timeSearch(column, 15, 0, values.length)
     );
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/QueryableIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/QueryableIndexStorageAdapterTest.java
index ddb1cc419f3..4cde2a79658 100644
--- a/processing/src/test/java/org/apache/druid/segment/QueryableIndexStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/QueryableIndexStorageAdapterTest.java
@@ -20,12 +20,7 @@
 package org.apache.druid.segment;
 
 import org.apache.druid.hll.HyperLogLogCollector;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Yielder;
-import org.apache.druid.java.util.common.guava.Yielders;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -74,18 +69,8 @@ public class QueryableIndexStorageAdapterTest
     {
       final QueryableIndex index = TestIndex.getMMappedTestIndex();
       final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index);
-
       if (vectorize) {
-        final VectorCursor cursor = closer.register(
-            adapter.makeVectorCursor(
-                null,
-                Intervals.ETERNITY,
-                VirtualColumns.EMPTY,
-                false,
-                QueryableIndexStorageAdapter.DEFAULT_VECTOR_SIZE,
-                null
-            )
-        );
+        final VectorCursor cursor = closer.register(adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)).asVectorCursor();
 
         final VectorColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
@@ -96,17 +81,8 @@ public class QueryableIndexStorageAdapterTest
         partialNullSelector =
             columnSelectorFactory.makeSingleValueDimensionSelector(DefaultDimensionSpec.of("partial_null_column"));
       } else {
-        final Sequence<Cursor> cursors = adapter.makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        );
-
-        final Yielder<Cursor> yielder = closer.register(Yielders.each(cursors));
-        final Cursor cursor = yielder.get();
+        final CursorHolder cursorHolder = closer.register(adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN));
+        final Cursor cursor = cursorHolder.asCursor();
         final ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
 
         qualitySelector =
@@ -250,18 +226,9 @@ public class QueryableIndexStorageAdapterTest
     {
       final QueryableIndex index = TestIndex.getMMappedTestIndex();
       final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index);
-      final Sequence<Cursor> cursors = adapter.makeCursors(
-          null,
-          Intervals.ETERNITY,
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
-      final Yielder<Cursor> cursorYielder = Yielders.each(cursors);
-      cursor = cursorYielder.get();
+      final CursorHolder cursorHolder = closer.register(adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN));
+      cursor = cursorHolder.asCursor();
       columnSelectorFactory = cursor.getColumnSelectorFactory();
-      closer.register(cursorYielder);
     }
 
     @After
diff --git a/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java
index 9822f6e5f33..dca89de18b1 100644
--- a/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java
@@ -19,18 +19,21 @@
 
 package org.apache.druid.segment;
 
+import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import com.google.common.math.LongMath;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.common.guava.GuavaUtils;
 import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.query.CursorGranularizer;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.segment.column.ColumnCapabilities;
@@ -38,16 +41,20 @@ import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
+import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+
+import java.math.RoundingMode;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.function.ToLongFunction;
@@ -78,10 +85,10 @@ public class RowBasedStorageAdapterTest
           }
       );
 
-  private static final List<Function<Cursor, Supplier<Object>>> READ_TIME_AND_STRING =
+  private static final List<BiFunction<Cursor, CursorGranularizer, Supplier<Object>>> READ_TIME_AND_STRING_GRAN =
       ImmutableList.of(
-          cursor -> cursor::getTime,
-          cursor -> {
+          (cursor, granularizer) -> granularizer::getBucketStart,
+          (cursor, granularizer) -> {
             final BaseObjectColumnValueSelector selector =
                 cursor.getColumnSelectorFactory().makeColumnValueSelector(ValueType.STRING.name());
             return selector::getObject;
@@ -98,11 +105,6 @@ public class RowBasedStorageAdapterTest
 
     PROCESSORS.clear();
 
-    PROCESSORS.put(
-        "cursor-time",
-        cursor -> cursor::getTime
-    );
-
     // Read all the types as all the other types.
 
     for (final String valueTypeName : ROW_SIGNATURE.getColumnNames()) {
@@ -173,7 +175,7 @@ public class RowBasedStorageAdapterTest
 
   /**
    * A RowAdapter for Integers where:
-   *
+   * <p>
    * 1) timestampFunction returns a timestamp where the millis instant is equal to that integer as a number of hours
    * since the epoch (1970).
    * 2) columnFunction provides columns named after value types where each one equal to the cast to that type. All
@@ -208,14 +210,52 @@ public class RowBasedStorageAdapterTest
         }
       };
 
+  private static final RowAdapter<Integer> SAME_TIME_ROW_ADAPTER =
+      new RowAdapter<Integer>()
+      {
+        private DateTime startTime = DateTimes.nowUtc();
+        @Override
+        public ToLongFunction<Integer> timestampFunction()
+        {
+          return i -> {
+            long div = LongMath.divide(i, 3, RoundingMode.FLOOR);
+            return startTime.plus(div).getMillis();
+          };
+        }
+
+        @Override
+        public Function<Integer, Object> columnFunction(String columnName)
+        {
+          if (UNKNOWN_TYPE_NAME.equals(columnName)) {
+            return i -> i;
+          } else {
+            final ValueType valueType = GuavaUtils.getEnumIfPresent(ValueType.class, columnName);
+
+            if (valueType == null || valueType == ValueType.COMPLEX) {
+              return i -> null;
+            } else {
+              return i -> DimensionHandlerUtils.convertObjectToType(
+                  i,
+                  ROW_SIGNATURE.getColumnType(columnName).orElse(null)
+              );
+            }
+          }
+        }
+      };
+
   public final AtomicLong numCloses = new AtomicLong();
 
   private RowBasedStorageAdapter<Integer> createIntAdapter(final int... ints)
+  {
+    return createIntAdapter(ROW_ADAPTER, ints);
+  }
+
+  private RowBasedStorageAdapter<Integer> createIntAdapter(RowAdapter<Integer> adapter, final int... ints)
   {
     return new RowBasedStorageAdapter<>(
         Sequences.simple(Arrays.stream(ints).boxed().collect(Collectors.toList()))
                  .withBaggage(numCloses::incrementAndGet),
-        ROW_ADAPTER,
+        adapter,
         ROW_SIGNATURE
     );
   }
@@ -262,7 +302,7 @@ public class RowBasedStorageAdapterTest
   {
     final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
 
-    // Row based adapters don't know cardinality (they don't walk their Iterables until makeCursors is called).
+    // Row based adapters don't know cardinality (they don't walk their Iterables until makeCursor is called).
     for (String column : ROW_SIGNATURE.getColumnNames()) {
       Assert.assertEquals(DimensionDictionarySelector.CARDINALITY_UNKNOWN, adapter.getDimensionCardinality(column));
     }
@@ -331,7 +371,7 @@ public class RowBasedStorageAdapterTest
   {
     final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
 
-    // Row based adapters don't know cardinality (they don't walk their Iterables until makeCursors is called).
+    // Row based adapters don't know cardinality (they don't walk their Iterables until makeCursor is called).
     for (String column : ROW_SIGNATURE.getColumnNames()) {
       Assert.assertEquals(DimensionDictionarySelector.CARDINALITY_UNKNOWN, adapter.getDimensionCardinality(column));
     }
@@ -451,418 +491,447 @@ public class RowBasedStorageAdapterTest
   }
 
   @Test
-  public void test_makeCursors_filterOnLong()
+  public void test_makeCursor()
   {
     final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
 
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        new SelectorDimFilter(ValueType.LONG.name(), "1.0", null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            ImmutableList.of("1")
-        ),
-        walkCursors(cursors, READ_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_filterOnNonexistentColumnEqualsNull()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        new SelectorDimFilter("nonexistent", null, null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            ImmutableList.of("0"),
-            ImmutableList.of("1")
-        ),
-        walkCursors(cursors, READ_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_filterOnVirtualColumn()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        new SelectorDimFilter("vc", "2", null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.create(
-            ImmutableList.of(
-                new ExpressionVirtualColumn(
-                    "vc",
-                    "\"LONG\" + 1",
-                    ColumnType.LONG,
-                    ExprMacroTable.nil()
-                )
-            )
-        ),
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            ImmutableList.of("1")
-        ),
-        walkCursors(cursors, READ_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_descending()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        true,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            ImmutableList.of("2"),
-            ImmutableList.of("1"),
-            ImmutableList.of("0")
-        ),
-        walkCursors(cursors, READ_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_intervalDoesNotMatch()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        Intervals.of("2000/P1D"),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(),
-        walkCursors(cursors, READ_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_intervalPartiallyMatches()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        Intervals.of("1970-01-01T01/PT1H"),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            ImmutableList.of("1")
-        ),
-        walkCursors(cursors, READ_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_hourGranularity()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 1, 2, 3);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        Intervals.of("1970/1971"),
-        VirtualColumns.EMPTY,
-        Granularities.HOUR,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            ImmutableList.of(DateTimes.of("1970-01-01T00"), "0"),
-            ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
-            ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
-            ImmutableList.of(DateTimes.of("1970-01-01T02"), "2"),
-            ImmutableList.of(DateTimes.of("1970-01-01T03"), "3")
-        ),
-        walkCursors(cursors, READ_TIME_AND_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_hourGranularityWithInterval()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 1, 2, 3);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        Intervals.of("1970-01-01T01/PT2H"),
-        VirtualColumns.EMPTY,
-        Granularities.HOUR,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
-            ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
-            ImmutableList.of(DateTimes.of("1970-01-01T02"), "2")
-        ),
-        walkCursors(cursors, READ_TIME_AND_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_hourGranularityWithIntervalDescending()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 1, 2, 3);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        Intervals.of("1970-01-01T01/PT2H"),
-        VirtualColumns.EMPTY,
-        Granularities.HOUR,
-        true,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            ImmutableList.of(DateTimes.of("1970-01-01T02"), "2"),
-            ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
-            ImmutableList.of(DateTimes.of("1970-01-01T01"), "1")
-        ),
-        walkCursors(cursors, READ_TIME_AND_STRING)
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_allProcessors()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(
-            Lists.newArrayList(
-                Intervals.ETERNITY.getStart(),
-
-                // FLOAT
-                0f,
-                0d,
-                0L,
-                "0.0",
-                0f,
-
-                // DOUBLE
-                0f,
-                0d,
-                0L,
-                "0.0",
-                0d,
-
-                // LONG
-                0f,
-                0d,
-                0L,
-                "0",
-                0L,
-
-                // STRING
-                0f,
-                0d,
-                0L,
-                "0",
-                "0",
-
-                // COMPLEX
-                NullHandling.defaultFloatValue(),
-                NullHandling.defaultDoubleValue(),
-                NullHandling.defaultLongValue(),
-                null,
-                null,
-
-                // unknownType
-                0f,
-                0d,
-                0L,
-                "0",
-                0
-            ),
-            Lists.newArrayList(
-                Intervals.ETERNITY.getStart(),
-
-                // FLOAT
-                1f,
-                1d,
-                1L,
-                "1.0",
-                1f,
-
-                // DOUBLE
-                1f,
-                1d,
-                1L,
-                "1.0",
-                1d,
-
-                // LONG
-                1f,
-                1d,
-                1L,
-                "1",
-                1L,
-
-                // STRING
-                1f,
-                1d,
-                1L,
-                "1",
-                "1",
-
-                // COMPLEX
-                NullHandling.defaultFloatValue(),
-                NullHandling.defaultDoubleValue(),
-                NullHandling.defaultLongValue(),
-                null,
-                null,
-
-                // unknownType
-                1f,
-                1d,
-                1L,
-                "1",
-                1
-            )
-        ),
-        walkCursors(cursors, new ArrayList<>(PROCESSORS.values()))
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_filterOnNonexistentColumnEqualsNonnull()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
-
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        new SelectorDimFilter("nonexistent", "abc", null).toFilter(),
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    Assert.assertEquals(
-        ImmutableList.of(),
-        walkCursors(cursors, new ArrayList<>(PROCESSORS.values()))
-    );
-
-    Assert.assertEquals(1, numCloses.get());
-  }
-
-  @Test
-  public void test_makeCursors_eternityIntervalWithMonthGranularity()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
-    Assert.assertThrows(IAE.class, () -> {
-      adapter.makeCursors(
-          null,
-          Intervals.ETERNITY,
-          VirtualColumns.EMPTY,
-          Granularities.MONTH,
-          false,
-          null
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .build();
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of("0"),
+              ImmutableList.of("1"),
+              ImmutableList.of("2")
+          ),
+          walkCursor(cursor, READ_STRING)
       );
-    });
+    }
+
+    Assert.assertEquals(2, numCloses.get());
   }
 
-  private static List<List<Object>> walkCursors(
-      final Sequence<Cursor> cursors,
+  @Test
+  public void test_makeCursor_filterOnLong()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(new SelectorDimFilter(ValueType.LONG.name(), "1.0", null).toFilter())
+                                                     .build();
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of("1")
+          ),
+          walkCursor(cursor, READ_STRING)
+      );
+    }
+
+
+    Assert.assertEquals(2, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_filterOnNonexistentColumnEqualsNull()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(new SelectorDimFilter("nonexistent", null, null).toFilter())
+                                                     .build();
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of("0"),
+              ImmutableList.of("1")
+          ),
+          walkCursor(cursor, READ_STRING)
+      );
+    }
+
+    Assert.assertEquals(2, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_filterOnVirtualColumn()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(new SelectorDimFilter("vc", "2", null).toFilter())
+                                                     .setVirtualColumns(
+                                                         VirtualColumns.create(
+                                                             ImmutableList.of(
+                                                                 new ExpressionVirtualColumn(
+                                                                     "vc",
+                                                                     "\"LONG\" + 1",
+                                                                     ColumnType.LONG,
+                                                                     ExprMacroTable.nil()
+                                                                 )
+                                                             )
+                                                         )
+                                                     )
+                                                     .build();
+
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of("1")
+          ),
+          walkCursor(cursor, READ_STRING)
+      );
+    }
+
+    Assert.assertEquals(2, numCloses.get());
+  }
+
+
+  @Test
+  public void test_makeCursor_descending()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setPreferredOrdering(Cursors.descendingTimeOrder())
+                                                     .build();
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of("2"),
+              ImmutableList.of("1"),
+              ImmutableList.of("0")
+          ),
+          walkCursor(cursor, READ_STRING)
+      );
+    }
+
+    Assert.assertEquals(1, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_intervalDoesNotMatch()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setInterval(Intervals.of("2000/P1D"))
+                                                     .build();
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(),
+          walkCursor(cursor, READ_STRING)
+      );
+    }
+
+    Assert.assertEquals(2, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_intervalPartiallyMatches()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setInterval(Intervals.of("1970-01-01T01/PT1H"))
+                                                     .build();
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of("1")
+          ),
+          walkCursor(cursor, READ_STRING)
+      );
+    }
+
+    Assert.assertEquals(2, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_hourGranularity()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 1, 2, 3);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setInterval(Intervals.of("1970/1971"))
+                                                     .build();
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of(DateTimes.of("1970-01-01T00"), "0"),
+              ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
+              ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
+              ImmutableList.of(DateTimes.of("1970-01-01T02"), "2"),
+              ImmutableList.of(DateTimes.of("1970-01-01T03"), "3")
+          ),
+          walkCursorGranularized(adapter, cursor, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
+      );
+    }
+
+    Assert.assertEquals(1, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_hourGranularityWithInterval()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 1, 2, 3);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setInterval(Intervals.of("1970-01-01T01/PT2H"))
+                                                     .build();
+
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
+              ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
+              ImmutableList.of(DateTimes.of("1970-01-01T02"), "2")
+          ),
+          walkCursorGranularized(adapter, cursor, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
+      );
+    }
+
+    Assert.assertEquals(1, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_hourGranularityWithIntervalDescending()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 1, 2, 3);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setInterval(Intervals.of("1970-01-01T01/PT2H"))
+                                                     .setPreferredOrdering(Cursors.descendingTimeOrder())
+                                                     .build();
+
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(
+              ImmutableList.of(DateTimes.of("1970-01-01T02"), "2"),
+              ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
+              ImmutableList.of(DateTimes.of("1970-01-01T01"), "1")
+          ),
+          walkCursorGranularized(adapter, cursor, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
+      );
+    }
+
+    Assert.assertEquals(1, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_allProcessors()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
+
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(
+              Lists.newArrayList(
+
+                  // FLOAT
+                  0f,
+                  0d,
+                  0L,
+                  "0.0",
+                  0f,
+
+                  // DOUBLE
+                  0f,
+                  0d,
+                  0L,
+                  "0.0",
+                  0d,
+
+                  // LONG
+                  0f,
+                  0d,
+                  0L,
+                  "0",
+                  0L,
+
+                  // STRING
+                  0f,
+                  0d,
+                  0L,
+                  "0",
+                  "0",
+
+                  // COMPLEX
+                  NullHandling.defaultFloatValue(),
+                  NullHandling.defaultDoubleValue(),
+                  NullHandling.defaultLongValue(),
+                  null,
+                  null,
+
+                  // unknownType
+                  0f,
+                  0d,
+                  0L,
+                  "0",
+                  0
+              ),
+              Lists.newArrayList(
+
+                  // FLOAT
+                  1f,
+                  1d,
+                  1L,
+                  "1.0",
+                  1f,
+
+                  // DOUBLE
+                  1f,
+                  1d,
+                  1L,
+                  "1.0",
+                  1d,
+
+                  // LONG
+                  1f,
+                  1d,
+                  1L,
+                  "1",
+                  1L,
+
+                  // STRING
+                  1f,
+                  1d,
+                  1L,
+                  "1",
+                  "1",
+
+                  // COMPLEX
+                  NullHandling.defaultFloatValue(),
+                  NullHandling.defaultDoubleValue(),
+                  NullHandling.defaultLongValue(),
+                  null,
+                  null,
+
+                  // unknownType
+                  1f,
+                  1d,
+                  1L,
+                  "1",
+                  1
+              )
+          ),
+          walkCursor(cursor, new ArrayList<>(PROCESSORS.values()))
+      );
+    }
+
+    Assert.assertEquals(2, numCloses.get());
+  }
+
+  @Test
+  public void test_makeCursor_filterOnNonexistentColumnEqualsNonnull()
+  {
+    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
+
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(new SelectorDimFilter("nonexistent", "abc", null).toFilter())
+                                                     .build();
+
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Assert.assertEquals(
+          ImmutableList.of(),
+          walkCursor(cursor, new ArrayList<>(PROCESSORS.values()))
+      );
+    }
+
+    Assert.assertEquals(2, numCloses.get());
+  }
+
+  private static List<List<Object>> walkCursor(
+      final Cursor cursor,
       final List<Function<Cursor, Supplier<Object>>> processors
   )
   {
-    return cursors.flatMap(
-        cursor -> {
-          // Gather test-value suppliers together.
-          final List<Supplier<Object>> suppliers = new ArrayList<>();
-          for (Function<Cursor, Supplier<Object>> processor : processors) {
-            suppliers.add(processor.apply(cursor));
-          }
+    final List<Supplier<Object>> suppliers = new ArrayList<>();
+    for (Function<Cursor, Supplier<Object>> processor : processors) {
+      suppliers.add(processor.apply(cursor));
+    }
 
-          final List<List<Object>> retVal = new ArrayList<>();
+    final List<List<Object>> retVal = new ArrayList<>();
 
-          while (!cursor.isDone()) {
-            final List<Object> row = new ArrayList<>();
+    // test cursor reset
+    while (!cursor.isDone()) {
+      cursor.advanceUninterruptibly();
+    }
 
-            for (Supplier<Object> supplier : suppliers) {
-              row.add(supplier.get());
-            }
+    cursor.reset();
 
-            retVal.add(row);
-            cursor.advanceUninterruptibly();
-          }
+    while (!cursor.isDone()) {
+      final List<Object> row = new ArrayList<>();
 
-          return Sequences.simple(retVal);
-        }
-    ).toList();
+      for (Supplier<Object> supplier : suppliers) {
+        row.add(supplier.get());
+      }
+
+      retVal.add(row);
+      cursor.advanceUninterruptibly();
+    }
+
+    return retVal;
+  }
+
+  private static List<List<Object>> walkCursorGranularized(
+      final StorageAdapter adapter,
+      final Cursor cursor,
+      final CursorBuildSpec buildSpec,
+      final Granularity granularity,
+      final List<BiFunction<Cursor, CursorGranularizer, Supplier<Object>>> processors
+  )
+  {
+    CursorGranularizer granularizer = CursorGranularizer.create(
+        adapter,
+        cursor,
+        granularity,
+        buildSpec.getInterval(),
+        Cursors.preferDescendingTimeOrdering(buildSpec)
+    );
+
+    final List<Supplier<Object>> suppliers = new ArrayList<>();
+    for (BiFunction<Cursor, CursorGranularizer, Supplier<Object>> processor : processors) {
+      suppliers.add(processor.apply(cursor, granularizer));
+    }
+
+    final Sequence<List<Object>> theSequence =
+        Sequences.simple(granularizer.getBucketIterable())
+                 .flatMap(bucketInterval -> {
+                   if (!granularizer.advanceToBucket(bucketInterval)) {
+                     return Sequences.empty();
+                   }
+                   final List<List<Object>> retVal = new ArrayList<>();
+                   while (!cursor.isDone()) {
+                     final List<Object> row = new ArrayList<>();
+
+                     for (Supplier<Object> supplier : suppliers) {
+                       row.add(supplier.get());
+                     }
+
+                     retVal.add(row);
+                     if (!granularizer.advanceCursorWithinBucketUninterruptedly()) {
+                       break;
+                     }
+                   }
+                   return Sequences.simple(retVal);
+                 })
+                 .filter(Predicates.notNull());
+    return theSequence.toList();
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/TombstoneSegmentStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/TombstoneSegmentStorageAdapterTest.java
index c93abe864f1..8950693f14f 100644
--- a/processing/src/test/java/org/apache/druid/segment/TombstoneSegmentStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/TombstoneSegmentStorageAdapterTest.java
@@ -19,10 +19,6 @@
 
 package org.apache.druid.segment;
 
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.data.Indexed;
 import org.joda.time.DateTime;
@@ -41,15 +37,17 @@ public class TombstoneSegmentStorageAdapterTest
     StorageAdapter sa = new StorageAdapter()
     {
       @Override
-      public Sequence<Cursor> makeCursors(
-          @Nullable Filter filter,
-          Interval interval,
-          VirtualColumns virtualColumns,
-          Granularity gran,
-          boolean descending,
-          @Nullable QueryMetrics<?> queryMetrics)
+      public CursorHolder makeCursorHolder(CursorBuildSpec spec)
       {
-        return null;
+        return new CursorHolder()
+        {
+          @Nullable
+          @Override
+          public Cursor asCursor()
+          {
+            return null;
+          }
+        };
       }
 
       @Override
diff --git a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
index b467bc6c938..8ea5791463f 100644
--- a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
@@ -25,12 +25,10 @@ import org.apache.druid.data.input.ResourceInputSource;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.NestedDataTestUtils;
-import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.filter.EqualityFilter;
 import org.apache.druid.query.filter.Filter;
@@ -38,6 +36,7 @@ import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.filter.AndFilter;
 import org.apache.druid.segment.filter.OrFilter;
@@ -57,7 +56,6 @@ import org.apache.druid.timeline.partition.LinearShardSpec;
 import org.apache.druid.utils.CloseableUtils;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.MatcherAssert;
-import org.joda.time.Interval;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -65,10 +63,12 @@ import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
-import javax.annotation.Nullable;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Function;
+import java.util.function.ToLongFunction;
 
 import static org.apache.druid.segment.filter.FilterTestUtils.not;
 import static org.apache.druid.segment.filter.FilterTestUtils.selector;
@@ -233,26 +233,24 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
   @Test
   public void test_unnest_adapters_basic()
   {
+    try (final CursorHolder cursorHolder = UNNEST_STORAGE_ADAPTER.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
 
-    Sequence<Cursor> cursorSequence = UNNEST_STORAGE_ADAPTER.makeCursors(
-        null,
-        UNNEST_STORAGE_ADAPTER.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    cursorSequence.accumulate(null, (accumulated, cursor) -> {
       ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
 
       DimensionSelector dimSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(OUTPUT_COLUMN_NAME));
       int count = 0;
+
+      List<Object> rows = new ArrayList<>();
+      // test cursor reset
+      while (!cursor.isDone()) {
+        cursor.advance();
+      }
+      cursor.reset();
+
       while (!cursor.isDone()) {
         Object dimSelectorVal = dimSelector.getObject();
-        if (dimSelectorVal == null) {
-          Assert.assertNull(dimSelectorVal);
-        }
+        rows.add(dimSelectorVal);
         cursor.advance();
         count++;
       }
@@ -261,25 +259,115 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
       unnest 2 rows -> 16 rows after unnest
        */
       Assert.assertEquals(count, 16);
-      return null;
-    });
+      Assert.assertEquals(
+          Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "10", "11", "12", "13", "14", "15", "8", "9"),
+          rows
+      );
+    }
+  }
 
+  @Test
+  public void test_unnest_adapters_basic_array_column()
+  {
+    try (final CursorHolder cursorHolder = UNNEST_ARRAYS.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+
+      ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
+
+      ColumnValueSelector dimSelector = factory.makeColumnValueSelector("u");
+      int count = 0;
+
+      List<Object> rows = new ArrayList<>();
+      // test cursor reset
+      while (!cursor.isDone()) {
+        cursor.advance();
+      }
+      cursor.reset();
+
+      while (!cursor.isDone()) {
+        Object dimSelectorVal = dimSelector.getObject();
+        rows.add(dimSelectorVal);
+        cursor.advance();
+        count++;
+      }
+      Assert.assertEquals(count, 12);
+      Assert.assertEquals(
+          Arrays.asList(2L, 3L, 1L, null, 3L, 1L, null, 2L, 9L, 1L, 2L, 3L),
+          rows
+      );
+    }
+  }
+
+  @Test
+  public void test_unnest_adapters_basic_row_based_array_column()
+  {
+    StorageAdapter adapter = new UnnestStorageAdapter(
+        new RowBasedStorageAdapter<>(
+            Sequences.simple(
+                Arrays.asList(
+                    new Object[]{1L, new Object[]{1L, 2L}},
+                    new Object[]{1L, new Object[]{3L, 4L, 5L}},
+                    new Object[]{2L, new Object[]{6L, null, 7L}},
+                    new Object[]{2L, null},
+                    new Object[]{3L, new Object[]{8L, 9L, 10L}}
+                )
+            ),
+            new RowAdapter<Object[]>()
+            {
+              @Override
+              public ToLongFunction<Object[]> timestampFunction()
+              {
+                return value -> (long) value[0];
+              }
+
+              @Override
+              public Function<Object[], Object> columnFunction(String columnName)
+              {
+                if ("a".equals(columnName)) {
+                  return objects -> objects[1];
+                }
+                return null;
+              }
+            },
+            RowSignature.builder().add("arrayLongNulls", ColumnType.LONG_ARRAY).build()
+        ),
+        new ExpressionVirtualColumn("u", "\"a\"", ColumnType.LONG, ExprMacroTable.nil()),
+        null
+    );
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+
+      ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
+
+      ColumnValueSelector dimSelector = factory.makeColumnValueSelector("u");
+      int count = 0;
+
+      List<Object> rows = new ArrayList<>();
+      // test cursor reset
+      while (!cursor.isDone()) {
+        cursor.advance();
+      }
+      cursor.reset();
+
+      while (!cursor.isDone()) {
+        Object dimSelectorVal = dimSelector.getObject();
+        rows.add(dimSelectorVal);
+        cursor.advance();
+        count++;
+      }
+      Assert.assertEquals(count, 11);
+      Assert.assertEquals(
+          Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, null, 7L, 8L, 9L, 10L),
+          rows
+      );
+    }
   }
 
   @Test
   public void test_two_levels_of_unnest_adapters()
   {
-    Sequence<Cursor> cursorSequence = UNNEST_STORAGE_ADAPTER1.makeCursors(
-        null,
-        UNNEST_STORAGE_ADAPTER1.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-
-    cursorSequence.accumulate(null, (accumulated, cursor) -> {
+    try (final CursorHolder cursorHolder = UNNEST_STORAGE_ADAPTER1.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
       ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
 
       DimensionSelector dimSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(OUTPUT_COLUMN_NAME1));
@@ -305,8 +393,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
        */
       Assert.assertEquals(count, 128);
       Assert.assertEquals(dimSelector.getValueCardinality(), 17);
-      return null;
-    });
+    }
   }
 
   @Test
@@ -332,26 +419,20 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         selector(inputColumn, "2")
     ));
 
-    final Sequence<Cursor> cursorSequence = unnestStorageAdapter.makeCursors(
-        baseFilter,
-        unnestStorageAdapter.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(baseFilter)
+                                                     .build();
+    try (final CursorHolder cursorHolder = unnestStorageAdapter.makeCursorHolder(buildSpec)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final TestStorageAdapter base = (TestStorageAdapter) unnestStorageAdapter.getBaseAdapter();
+      final Filter pushDownFilter = base.getPushDownFilter();
 
-    final TestStorageAdapter base = (TestStorageAdapter) unnestStorageAdapter.getBaseAdapter();
-    final Filter pushDownFilter = base.getPushDownFilter();
-
-    Assert.assertEquals(expectedPushDownFilter, pushDownFilter);
-    cursorSequence.accumulate(null, (accumulated, cursor) -> {
+      Assert.assertEquals(expectedPushDownFilter, pushDownFilter);
       Assert.assertEquals(cursor.getClass(), PostJoinCursor.class);
       final Filter postFilter = ((PostJoinCursor) cursor).getPostJoinFilter();
       // OR-case so base filter should match the postJoinFilter
       Assert.assertEquals(baseFilter, postFilter);
-      return null;
-    });
+    }
   }
 
   @Test
@@ -383,27 +464,22 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         ))
     ));
 
-    final Sequence<Cursor> cursorSequence = unnestStorageAdapter.makeCursors(
-        baseFilter,
-        unnestStorageAdapter.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(baseFilter)
+                                                     .build();
+    try (final CursorHolder cursorHolder = unnestStorageAdapter.makeCursorHolder(buildSpec)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final TestStorageAdapter base = (TestStorageAdapter) unnestStorageAdapter.getBaseAdapter();
+      final Filter pushDownFilter = base.getPushDownFilter();
 
-    final TestStorageAdapter base = (TestStorageAdapter) unnestStorageAdapter.getBaseAdapter();
-    final Filter pushDownFilter = base.getPushDownFilter();
-
-    Assert.assertEquals(expectedPushDownFilter, pushDownFilter);
-    cursorSequence.accumulate(null, (accumulated, cursor) -> {
+      Assert.assertEquals(expectedPushDownFilter, pushDownFilter);
       Assert.assertEquals(cursor.getClass(), PostJoinCursor.class);
       final Filter postFilter = ((PostJoinCursor) cursor).getPostJoinFilter();
       // OR-case so base filter should match the postJoinFilter
       Assert.assertEquals(baseFilter, postFilter);
-      return null;
-    });
+    }
   }
+
   @Test
   public void test_nested_filters_unnested_and_topLevel1And3filtersInOR()
   {
@@ -421,6 +497,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         "(unnested-multi-string1 = 3 && (newcol = 2 || multi-string1 = 2 || unnested-multi-string1 = 1))"
     );
   }
+
   @Test
   public void test_nested_multiLevel_filters_unnested()
   {
@@ -444,6 +521,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         "(unnested-multi-string1 = 3 && (newcol = 2 || multi-string1 = 2 || (newcol = 3 && multi-string1 = 7) || unnested-multi-string1 = 1))"
     );
   }
+
   @Test
   public void test_nested_multiLevel_filters_unnested5Level()
   {
@@ -470,6 +548,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         "(unnested-multi-string1 = 3 || newcol = 2 || multi-string1 = 2 || (newcol = 3 && multi-string1 = 7 && newcol_1 = 10) || unnested-multi-string1 = 1)"
     );
   }
+
   @Test
   public void test_nested_filters_unnested_and_topLevelORAnd3filtersInOR()
   {
@@ -487,6 +566,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         "(unnested-multi-string1 = 3 || (newcol = 2 && multi-string1 = 2 && unnested-multi-string1 = 1))"
     );
   }
+
   @Test
   public void test_nested_filters_unnested_and_topLevelAND3filtersInORWithNestedOrs()
   {
@@ -652,7 +732,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
     final UnnestStorageAdapter unnestStorageAdapter = new UnnestStorageAdapter(
         new TestStorageAdapter(INCREMENTAL_INDEX),
         new ExpressionVirtualColumn(OUTPUT_COLUMN_NAME, "\"" + INPUT_COLUMN_NAME + "\"", null, ExprMacroTable.nil()),
-         new SelectorDimFilter(OUTPUT_COLUMN_NAME, "1", null)
+        new SelectorDimFilter(OUTPUT_COLUMN_NAME, "1", null)
     );
 
     final VirtualColumn vc = unnestStorageAdapter.getUnnestColumn();
@@ -662,21 +742,12 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
     final Filter expectedPushDownFilter =
         selector(inputColumn, "1");
 
+    try (final CursorHolder cursorHolder = unnestStorageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final TestStorageAdapter base = (TestStorageAdapter) unnestStorageAdapter.getBaseAdapter();
+      final Filter pushDownFilter = base.getPushDownFilter();
 
-    final Sequence<Cursor> cursorSequence = unnestStorageAdapter.makeCursors(
-        null,
-        unnestStorageAdapter.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    final TestStorageAdapter base = (TestStorageAdapter) unnestStorageAdapter.getBaseAdapter();
-    final Filter pushDownFilter = base.getPushDownFilter();
-
-    Assert.assertEquals(expectedPushDownFilter, pushDownFilter);
-    cursorSequence.accumulate(null, (accumulated, cursor) -> {
+      Assert.assertEquals(expectedPushDownFilter, pushDownFilter);
       Assert.assertEquals(cursor.getClass(), PostJoinCursor.class);
       final Filter postFilter = ((PostJoinCursor) cursor).getPostJoinFilter();
       Assert.assertEquals(unnestStorageAdapter.getUnnestFilter(), postFilter);
@@ -687,8 +758,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         count++;
       }
       Assert.assertEquals(1, count);
-      return null;
-    });
+    }
   }
 
 
@@ -709,20 +779,16 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         selector(inputColumn, "1");
 
     final Filter queryFilter = new SelectorFilter(OUTPUT_COLUMN_NAME, "1", null);
-    final Sequence<Cursor> cursorSequence = unnestStorageAdapter.makeCursors(
-        queryFilter,
-        unnestStorageAdapter.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(queryFilter)
+                                                     .build();
 
-    final TestStorageAdapter base = (TestStorageAdapter) unnestStorageAdapter.getBaseAdapter();
-    final Filter pushDownFilter = base.getPushDownFilter();
+    try (final CursorHolder cursorHolder = unnestStorageAdapter.makeCursorHolder(buildSpec)) {
+      Cursor cursor = cursorHolder.asCursor();
+      final TestStorageAdapter base = (TestStorageAdapter) unnestStorageAdapter.getBaseAdapter();
+      final Filter pushDownFilter = base.getPushDownFilter();
 
-    Assert.assertEquals(expectedPushDownFilter, pushDownFilter);
-    cursorSequence.accumulate(null, (accumulated, cursor) -> {
+      Assert.assertEquals(expectedPushDownFilter, pushDownFilter);
       Assert.assertEquals(cursor.getClass(), PostJoinCursor.class);
       final Filter postFilter = ((PostJoinCursor) cursor).getPostJoinFilter();
       Assert.assertEquals(queryFilter, postFilter);
@@ -733,8 +799,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         count++;
       }
       Assert.assertEquals(1, count);
-      return null;
-    });
+    }
   }
 
   @Test
@@ -761,16 +826,9 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         new ExpressionVirtualColumn(OUTPUT_COLUMN_NAME, "\"" + inputColumn + "\"", null, ExprMacroTable.nil()),
         null
     );
-    Sequence<Cursor> cursorSequence = withNullsStorageAdapter.makeCursors(
-        null,
-        withNullsStorageAdapter.getInterval(),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
 
-    cursorSequence.accumulate(null, (accumulated, cursor) -> {
+    try (final CursorHolder cursorHolder = withNullsStorageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
       ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
 
       DimensionSelector dimSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(OUTPUT_COLUMN_NAME));
@@ -788,9 +846,7 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
         count++;
       }
       Assert.assertEquals(count, 618);
-      return null;
-    });
-
+    }
   }
 
   public void testComputeBaseAndPostUnnestFilters(
@@ -866,16 +922,9 @@ class TestStorageAdapter extends IncrementalIndexStorageAdapter
   }
 
   @Override
-  public Sequence<Cursor> makeCursors(
-      @Nullable final Filter filter,
-      final Interval interval,
-      final VirtualColumns virtualColumns,
-      final Granularity gran,
-      final boolean descending,
-      @Nullable QueryMetrics<?> queryMetrics
-  )
+  public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    this.pushDownFilter = filter;
-    return super.makeCursors(filter, interval, virtualColumns, gran, descending, queryMetrics);
+    this.pushDownFilter = spec.getFilter();
+    return super.makeCursorHolder(spec);
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
index 179e4e96d10..1491fbd9d38 100644
--- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
@@ -44,16 +44,14 @@ import org.apache.druid.frame.segment.FrameStorageAdapter;
 import org.apache.druid.guice.BuiltInTypesModule;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprType;
 import org.apache.druid.math.expr.ExpressionType;
 import org.apache.druid.math.expr.Parser;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.aggregation.Aggregator;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
@@ -70,6 +68,8 @@ import org.apache.druid.segment.AutoTypeColumnSchema;
 import org.apache.druid.segment.ColumnInspector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IndexBuilder;
 import org.apache.druid.segment.IndexSpec;
@@ -416,11 +416,13 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
 
   protected StorageAdapter adapter;
 
+  protected VirtualColumns virtualColumns;
+
   // JUnit creates a new test instance for every test method call.
   // For filter tests, the test setup creates a segment.
   // Creating a new segment for every test method call is pretty slow, so cache the StorageAdapters.
   // Each thread gets its own map.
-  private static ThreadLocal<Map<String, Map<String, Pair<StorageAdapter, Closeable>>>> adapterCache =
+  private static ThreadLocal<Map<String, Map<String, AdapterStuff>>> adapterCache =
       ThreadLocal.withInitial(HashMap::new);
 
   public BaseFilterTest(
@@ -447,32 +449,40 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
   {
     BuiltInTypesModule.registerHandlersAndSerde();
     String className = getClass().getName();
-    Map<String, Pair<StorageAdapter, Closeable>> adaptersForClass = adapterCache.get().get(className);
+    Map<String, AdapterStuff> adaptersForClass = adapterCache.get().get(className);
     if (adaptersForClass == null) {
       adaptersForClass = new HashMap<>();
       adapterCache.get().put(className, adaptersForClass);
     }
 
-    Pair<StorageAdapter, Closeable> pair = adaptersForClass.get(testName);
-    if (pair == null) {
-      pair = finisher.apply(
+    AdapterStuff adapterStuff = adaptersForClass.get(testName);
+    if (adapterStuff == null) {
+      Pair<StorageAdapter, Closeable> pair = finisher.apply(
           indexBuilder.tmpDir(temporaryFolder.newFolder()).rows(rows)
       );
-      adaptersForClass.put(testName, pair);
+      adapterStuff = new AdapterStuff(
+          pair.lhs,
+          VirtualColumns.create(
+              Arrays.stream(VIRTUAL_COLUMNS.getVirtualColumns())
+                    .filter(x -> x.canVectorize(VIRTUAL_COLUMNS.wrapInspector(pair.lhs)))
+                    .collect(Collectors.toList())
+          ),
+          pair.rhs
+      );
+      adaptersForClass.put(testName, adapterStuff);
     }
 
-    this.adapter = pair.lhs;
-
+    this.adapter = adapterStuff.adapter;
+    this.virtualColumns = adapterStuff.virtualColumns;
   }
 
   public static void tearDown(String className) throws Exception
   {
-    Map<String, Pair<StorageAdapter, Closeable>> adaptersForClass = adapterCache.get().get(className);
+    Map<String, AdapterStuff> adaptersForClass = adapterCache.get().get(className);
 
     if (adaptersForClass != null) {
-      for (Map.Entry<String, Pair<StorageAdapter, Closeable>> entry : adaptersForClass.entrySet()) {
-        Closeable closeable = entry.getValue().rhs;
-        closeable.close();
+      for (Map.Entry<String, AdapterStuff> entry : adaptersForClass.entrySet()) {
+        entry.getValue().closeable.close();
       }
       adapterCache.get().put(className, null);
     }
@@ -516,7 +526,7 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
                                   new IncrementalIndexSchema(
                                       schema.getMinTimestamp(),
                                       schema.getTimestampSpec(),
-                                      schema.getGran(),
+                                      schema.getQueryGranularity(),
                                       schema.getVirtualColumns(),
                                       schema.getDimensionsSpec().withDimensions(
                                           schema.getDimensionsSpec()
@@ -544,7 +554,7 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
                                   new IncrementalIndexSchema(
                                       schema.getMinTimestamp(),
                                       schema.getTimestampSpec(),
-                                      schema.getGran(),
+                                      schema.getQueryGranularity(),
                                       schema.getVirtualColumns(),
                                       schema.getDimensionsSpec().withDimensions(
                                           schema.getDimensionsSpec()
@@ -573,7 +583,7 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
                                           new IncrementalIndexSchema(
                                               schema.getMinTimestamp(),
                                               schema.getTimestampSpec(),
-                                              schema.getGran(),
+                                              schema.getQueryGranularity(),
                                               schema.getVirtualColumns(),
                                               schema.getDimensionsSpec().withDimensions(
                                                   schema.getDimensionsSpec()
@@ -652,7 +662,7 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
                               new IncrementalIndexSchema(
                                   schema.getMinTimestamp(),
                                   schema.getTimestampSpec(),
-                                  schema.getGran(),
+                                  schema.getQueryGranularity(),
                                   schema.getVirtualColumns(),
                                   schema.getDimensionsSpec().withDimensions(
                                       schema.getDimensionsSpec()
@@ -675,7 +685,7 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
                               new IncrementalIndexSchema(
                                   schema.getMinTimestamp(),
                                   schema.getTimestampSpec(),
-                                  schema.getGran(),
+                                  schema.getQueryGranularity(),
                                   schema.getVirtualColumns(),
                                   schema.getDimensionsSpec().withDimensions(
                                       schema.getDimensionsSpec()
@@ -780,29 +790,32 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
     return optimize ? dimFilter.optimize(false) : dimFilter;
   }
 
-  private Sequence<Cursor> makeCursorSequence(final Filter filter)
+  private CursorBuildSpec makeCursorBuildSpec(@Nullable Filter filter)
   {
-    return adapter.makeCursors(
-        filter,
-        Intervals.ETERNITY,
-        VIRTUAL_COLUMNS,
-        Granularities.ALL,
-        false,
-        null
-    );
+    return CursorBuildSpec.builder()
+                          .setFilter(filter)
+                          .setVirtualColumns(VIRTUAL_COLUMNS)
+                          .build();
+
+  }
+
+  private CursorBuildSpec makeVectorCursorBuildSpec(@Nullable Filter filter)
+  {
+    return CursorBuildSpec.builder()
+                          .setFilter(filter)
+                          .setVirtualColumns(virtualColumns)
+                          .setQueryContext(
+                              QueryContext.of(
+                                  ImmutableMap.of(QueryContexts.VECTOR_SIZE_KEY, 3)
+                              )
+                          )
+                          .build();
   }
 
   private VectorCursor makeVectorCursor(final Filter filter)
   {
-
-    return adapter.makeVectorCursor(
-        filter,
-        Intervals.ETERNITY,
-        VIRTUAL_COLUMNS,
-        false,
-        3, // Vector size smaller than the number of rows, to ensure we use more than one.
-        null
-    );
+    final CursorBuildSpec buildSpec = makeVectorCursorBuildSpec(filter);
+    return adapter.makeCursorHolder(buildSpec).asVectorCursor();
   }
 
   /**
@@ -810,48 +823,39 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
    */
   private List<String> selectColumnValuesMatchingFilter(final DimFilter filter, final String selectColumn)
   {
-    final Sequence<Cursor> cursors = makeCursorSequence(makeFilter(filter));
-    Sequence<List<String>> seq = Sequences.map(
-        cursors,
-        cursor -> {
-          final DimensionSelector selector = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(makeCursorBuildSpec(makeFilter(filter)))) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final DimensionSelector selector = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
 
-          final List<String> values = new ArrayList<>();
+      final List<String> values = new ArrayList<>();
 
-          while (!cursor.isDone()) {
-            IndexedInts row = selector.getRow();
-            Preconditions.checkState(row.size() == 1);
-            values.add(selector.lookupName(row.get(0)));
-            cursor.advance();
-          }
-
-          return values;
-        }
-    );
-    return seq.toList().get(0);
+      while (!cursor.isDone()) {
+        IndexedInts row = selector.getRow();
+        Preconditions.checkState(row.size() == 1);
+        values.add(selector.lookupName(row.get(0)));
+        cursor.advance();
+      }
+      return values;
+    }
   }
 
   private long selectCountUsingFilteredAggregator(final DimFilter filter)
   {
-    final Sequence<Cursor> cursors = makeCursorSequence(null);
-    Sequence<Aggregator> aggSeq = Sequences.map(
-        cursors,
-        cursor -> {
-          Aggregator agg = new FilteredAggregatorFactory(
-              new CountAggregatorFactory("count"),
-              maybeOptimize(filter)
-          ).factorize(cursor.getColumnSelectorFactory());
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(makeCursorBuildSpec(null))) {
+      final Cursor cursor = cursorHolder.asCursor();
+      Aggregator agg = new FilteredAggregatorFactory(
+          new CountAggregatorFactory("count"),
+          maybeOptimize(filter)
+      ).factorize(cursor.getColumnSelectorFactory());
 
-          for (; !cursor.isDone(); cursor.advance()) {
-            agg.aggregate();
-          }
+      for (; !cursor.isDone(); cursor.advance()) {
+        agg.aggregate();
+      }
 
-          return agg;
-        }
-    );
-    return aggSeq.toList().get(0).getLong();
+      return agg.getLong();
+    }
   }
 
   private long selectCountUsingVectorizedFilteredAggregator(final DimFilter dimFilter)
@@ -862,7 +866,9 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
         dimFilter
     );
 
-    try (final VectorCursor cursor = makeVectorCursor(null)) {
+
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(makeVectorCursorBuildSpec(null))) {
+      final VectorCursor cursor = cursorHolder.asVectorCursor();
       final FilteredAggregatorFactory aggregatorFactory = new FilteredAggregatorFactory(
           new CountAggregatorFactory("count"),
           maybeOptimize(dimFilter)
@@ -928,27 +934,23 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
       }
     };
 
-    final Sequence<Cursor> cursors = makeCursorSequence(postFilteringFilter);
-    Sequence<List<String>> seq = Sequences.map(
-        cursors,
-        cursor -> {
-          final DimensionSelector selector = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(makeCursorBuildSpec(postFilteringFilter))) {
+      final Cursor cursor = cursorHolder.asCursor();
+      final DimensionSelector selector = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
 
-          final List<String> values = new ArrayList<>();
+      final List<String> values = new ArrayList<>();
 
-          while (!cursor.isDone()) {
-            IndexedInts row = selector.getRow();
-            Preconditions.checkState(row.size() == 1);
-            values.add(selector.lookupName(row.get(0)));
-            cursor.advance();
-          }
+      while (!cursor.isDone()) {
+        IndexedInts row = selector.getRow();
+        Preconditions.checkState(row.size() == 1);
+        values.add(selector.lookupName(row.get(0)));
+        cursor.advance();
+      }
 
-          return values;
-        }
-    );
-    return seq.toList().get(0);
+      return values;
+    }
   }
 
   private List<String> selectColumnValuesMatchingFilterUsingVectorizedPostFiltering(
@@ -992,7 +994,8 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
       }
     };
 
-    try (final VectorCursor cursor = makeVectorCursor(postFilteringFilter)) {
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(makeVectorCursorBuildSpec(postFilteringFilter))) {
+      final VectorCursor cursor = cursorHolder.asVectorCursor();
       final SingleValueDimensionVectorSelector selector = cursor
           .getColumnSelectorFactory()
           .makeSingleValueDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
@@ -1016,7 +1019,8 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
       final String selectColumn
   )
   {
-    try (final VectorCursor cursor = makeVectorCursor(makeFilter(filter))) {
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(makeVectorCursorBuildSpec(makeFilter(filter)))) {
+      final VectorCursor cursor = cursorHolder.asVectorCursor();
       final SingleValueDimensionVectorSelector selector = cursor
           .getColumnSelectorFactory()
           .makeSingleValueDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
@@ -1042,7 +1046,8 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
   )
   {
     final Expr parsedIdentifier = Parser.parse(selectColumn, TestExprMacroTable.INSTANCE);
-    try (final VectorCursor cursor = makeVectorCursor(makeFilter(filter))) {
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(makeVectorCursorBuildSpec(makeFilter(filter)))) {
+      final VectorCursor cursor = cursorHolder.asVectorCursor();
 
       final ExpressionType outputType = parsedIdentifier.getOutputType(cursor.getColumnSelectorFactory());
       final List<String> values = new ArrayList<>();
@@ -1240,4 +1245,22 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
       );
     }
   }
+
+  private static class AdapterStuff
+  {
+    private final StorageAdapter adapter;
+    private final VirtualColumns virtualColumns;
+    private final Closeable closeable;
+
+    private AdapterStuff(
+        StorageAdapter adapter,
+        VirtualColumns virtualColumns,
+        Closeable closeable
+    )
+    {
+      this.adapter = adapter;
+      this.virtualColumns = virtualColumns;
+      this.closeable = closeable;
+    }
+  }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/filter/FloatAndDoubleFilteringTest.java b/processing/src/test/java/org/apache/druid/segment/filter/FloatAndDoubleFilteringTest.java
index f61ed0f14c0..c9717f18628 100644
--- a/processing/src/test/java/org/apache/druid/segment/filter/FloatAndDoubleFilteringTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/filter/FloatAndDoubleFilteringTest.java
@@ -22,10 +22,6 @@ package org.apache.druid.segment.filter;
 import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.DoubleDimensionSchema;
@@ -37,11 +33,9 @@ import org.apache.druid.data.input.impl.TimeAndDimsParseSpec;
 import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.js.JavaScriptConfig;
 import org.apache.druid.query.extraction.MapLookupExtractor;
 import org.apache.druid.query.filter.BoundDimFilter;
-import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.filter.InDimFilter;
 import org.apache.druid.query.filter.JavaScriptDimFilter;
 import org.apache.druid.query.filter.RegexDimFilter;
@@ -55,7 +49,6 @@ import org.apache.druid.segment.IndexBuilder;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.incremental.IncrementalIndexSchema;
 import org.junit.AfterClass;
-import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -66,7 +59,6 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 
 @RunWith(Parameterized.class)
 public class FloatAndDoubleFilteringTest extends BaseFilterTest
@@ -74,8 +66,6 @@ public class FloatAndDoubleFilteringTest extends BaseFilterTest
   private static final String FLOAT_COLUMN = "flt";
   private static final String DOUBLE_COLUMN = "dbl";
   private static final String TIMESTAMP_COLUMN = "ts";
-  private static int EXECUTOR_NUM_THREADS = 16;
-  private static int EXECUTOR_NUM_TASKS = 2000;
   private static final int NUM_FILTER_VALUES = 32;
 
   private static final InputRowParser<Map<String, Object>> PARSER = new MapInputRowParser(
@@ -149,13 +139,6 @@ public class FloatAndDoubleFilteringTest extends BaseFilterTest
     doTestFloatFilterWithExtractionFn(DOUBLE_COLUMN);
   }
 
-  @Test
-  public void testMultithreaded()
-  {
-    doTestMultithreaded(FLOAT_COLUMN);
-    doTestMultithreaded(DOUBLE_COLUMN);
-  }
-
   private void doTestFloatColumnFiltering(final String columnName)
   {
     assertFilterMatches(
@@ -352,90 +335,4 @@ public class FloatAndDoubleFilteringTest extends BaseFilterTest
         ImmutableList.of("2", "3", "4")
     );
   }
-
-  private void doTestMultithreaded(final String columnName)
-  {
-    assertFilterMatchesMultithreaded(
-        new SelectorDimFilter(columnName, "3", null),
-        ImmutableList.of("3")
-    );
-
-    assertFilterMatchesMultithreaded(
-        new SelectorDimFilter(columnName, "3.0", null),
-        ImmutableList.of("3")
-    );
-
-    assertFilterMatchesMultithreaded(
-        new InDimFilter(columnName, Arrays.asList("2", "4", "8"), null),
-        ImmutableList.of("2", "4")
-    );
-
-    assertFilterMatchesMultithreaded(
-        new InDimFilter(columnName, Arrays.asList("2.0", "4.0", "8.0"), null),
-        ImmutableList.of("2", "4")
-    );
-
-    // cross the hashing threshold to test hashset implementation, filter on even values
-    List<String> infilterValues = new ArrayList<>(NUM_FILTER_VALUES);
-    for (int i = 0; i < NUM_FILTER_VALUES; i++) {
-      infilterValues.add(String.valueOf(i * 2));
-    }
-    assertFilterMatchesMultithreaded(
-        new InDimFilter(columnName, infilterValues, null),
-        ImmutableList.of("2", "4", "6")
-    );
-
-    assertFilterMatches(
-        new BoundDimFilter(columnName, "2", "5", false, false, null, null, StringComparators.NUMERIC),
-        ImmutableList.of("2", "3", "4", "5")
-    );
-
-    assertFilterMatches(
-        new BoundDimFilter(columnName, "2.0", "5.0", false, false, null, null, StringComparators.NUMERIC),
-        ImmutableList.of("2", "3", "4", "5")
-    );
-  }
-
-  private void assertFilterMatchesMultithreaded(
-      final DimFilter filter,
-      final List<String> expectedRows
-  )
-  {
-    testWithExecutor(filter, expectedRows);
-  }
-
-  private Runnable makeFilterRunner(
-      final DimFilter filter,
-      final List<String> expectedRows
-  )
-  {
-    return () -> assertFilterMatches(filter, expectedRows);
-  }
-
-  private void testWithExecutor(
-      final DimFilter filter,
-      final List<String> expectedRows
-  )
-  {
-    ListeningExecutorService executor = MoreExecutors.listeningDecorator(
-        Execs.multiThreaded(EXECUTOR_NUM_THREADS, "FloatAndDoubleFilteringTest-%d")
-    );
-
-    List<ListenableFuture<?>> futures = new ArrayList<>();
-
-    for (int i = 0; i < EXECUTOR_NUM_TASKS; i++) {
-      Runnable runnable = makeFilterRunner(filter, expectedRows);
-      ListenableFuture fut = executor.submit(runnable);
-      futures.add(fut);
-    }
-
-    try {
-      Futures.allAsList(futures).get(60, TimeUnit.SECONDS);
-    }
-    catch (Exception ex) {
-      Assert.fail(ex.getMessage());
-    }
-
-    executor.shutdown();
-  }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/filter/LongFilteringTest.java b/processing/src/test/java/org/apache/druid/segment/filter/LongFilteringTest.java
index d24c599eceb..3fd1c55ebab 100644
--- a/processing/src/test/java/org/apache/druid/segment/filter/LongFilteringTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/filter/LongFilteringTest.java
@@ -22,10 +22,6 @@ package org.apache.druid.segment.filter;
 import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.InputRowParser;
@@ -34,12 +30,10 @@ import org.apache.druid.data.input.impl.TimeAndDimsParseSpec;
 import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.js.JavaScriptConfig;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.query.extraction.MapLookupExtractor;
 import org.apache.druid.query.filter.BoundDimFilter;
-import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.filter.InDimFilter;
 import org.apache.druid.query.filter.JavaScriptDimFilter;
 import org.apache.druid.query.filter.RegexDimFilter;
@@ -53,7 +47,6 @@ import org.apache.druid.segment.IndexBuilder;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.incremental.IncrementalIndexSchema;
 import org.junit.AfterClass;
-import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -64,15 +57,12 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 
 @RunWith(Parameterized.class)
 public class LongFilteringTest extends BaseFilterTest
 {
   private static final String LONG_COLUMN = "lng";
   private static final String TIMESTAMP_COLUMN = "ts";
-  private static int EXECUTOR_NUM_THREADS = 16;
-  private static int EXECUTOR_NUM_TASKS = 2000;
   private static final int NUM_FILTER_VALUES = 32;
 
   private static final InputRowParser<Map<String, Object>> PARSER = new MapInputRowParser(
@@ -375,81 +365,4 @@ public class LongFilteringTest extends BaseFilterTest
         ImmutableList.of("2", "3", "4")
     );
   }
-
-  @Test
-  public void testMultithreaded()
-  {
-    assertFilterMatchesMultithreaded(
-        new SelectorDimFilter(LONG_COLUMN, "3", null),
-        ImmutableList.of("3")
-    );
-
-    assertFilterMatchesMultithreaded(
-        new InDimFilter(LONG_COLUMN, Arrays.asList("2", "4", "8"), null),
-        ImmutableList.of("2", "4")
-    );
-
-    // cross the hashing threshold to test hashset implementation, filter on even values
-    List<String> infilterValues = new ArrayList<>(NUM_FILTER_VALUES);
-    for (int i = 0; i < NUM_FILTER_VALUES; i++) {
-      infilterValues.add(String.valueOf(i * 2));
-    }
-    assertFilterMatchesMultithreaded(
-        new InDimFilter(LONG_COLUMN, infilterValues, null),
-        ImmutableList.of("2", "4", "6")
-    );
-
-    assertFilterMatches(
-        new BoundDimFilter(LONG_COLUMN, "2", "5", false, false, null, null, StringComparators.NUMERIC),
-        ImmutableList.of("2", "3", "4", "5")
-    );
-  }
-
-  private void assertFilterMatchesMultithreaded(
-      final DimFilter filter,
-      final List<String> expectedRows
-  )
-  {
-    testWithExecutor(filter, expectedRows);
-  }
-
-  private Runnable makeFilterRunner(
-      final DimFilter filter,
-      final List<String> expectedRows
-  )
-  {
-    return new Runnable()
-    {
-      @Override
-      public void run()
-      {
-        assertFilterMatches(filter, expectedRows);
-      }
-    };
-  }
-
-  private void testWithExecutor(
-      final DimFilter filter,
-      final List<String> expectedRows
-  )
-  {
-    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Execs.multiThreaded(EXECUTOR_NUM_THREADS, "LongFilteringTest-%d"));
-
-    List<ListenableFuture<?>> futures = new ArrayList<>();
-
-    for (int i = 0; i < EXECUTOR_NUM_TASKS; i++) {
-      Runnable runnable = makeFilterRunner(filter, expectedRows);
-      ListenableFuture fut = executor.submit(runnable);
-      futures.add(fut);
-    }
-
-    try {
-      Futures.allAsList(futures).get(60, TimeUnit.SECONDS);
-    }
-    catch (Exception ex) {
-      Assert.fail(ex.getMessage());
-    }
-
-    executor.shutdown();
-  }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/filter/PredicateValueMatcherFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/filter/PredicateValueMatcherFactoryTest.java
index c2e7a491257..ff4dc11f220 100644
--- a/processing/src/test/java/org/apache/druid/segment/filter/PredicateValueMatcherFactoryTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/filter/PredicateValueMatcherFactoryTest.java
@@ -21,7 +21,6 @@ package org.apache.druid.segment.filter;
 
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.query.filter.SelectorPredicateFactory;
 import org.apache.druid.query.filter.ValueMatcher;
@@ -110,8 +109,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Float> columnValueSelector = TestColumnValueSelector.of(
         Float.class,
-        ImmutableList.of(2.f),
-        DateTimes.nowUtc()
+        ImmutableList.of(2.f)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("2.f").makeFloatProcessor(columnValueSelector);
@@ -123,8 +121,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Float> columnValueSelector = TestColumnValueSelector.of(
         Float.class,
-        ImmutableList.of(2.f),
-        DateTimes.nowUtc()
+        ImmutableList.of(2.f)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("5.f").makeFloatProcessor(columnValueSelector);
@@ -136,8 +133,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Double> columnValueSelector = TestColumnValueSelector.of(
         Double.class,
-        ImmutableList.of(2.),
-        DateTimes.nowUtc()
+        ImmutableList.of(2.)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("2.").makeDoubleProcessor(columnValueSelector);
@@ -149,8 +145,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Double> columnValueSelector = TestColumnValueSelector.of(
         Double.class,
-        ImmutableList.of(2.),
-        DateTimes.nowUtc()
+        ImmutableList.of(2.)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("5.").makeDoubleProcessor(columnValueSelector);
@@ -187,8 +182,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
               {
                 return num;
               }
-            }),
-            DateTimes.nowUtc()
+            })
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("2").makeComplexProcessor(columnValueSelector);
@@ -225,8 +219,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
               {
                 return num;
               }
-            }),
-            DateTimes.nowUtc()
+            })
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("5").makeComplexProcessor(columnValueSelector);
@@ -238,8 +231,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Long> columnValueSelector = TestColumnValueSelector.of(
         Long.class,
-        ImmutableList.of(2L),
-        DateTimes.nowUtc()
+        ImmutableList.of(2L)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("2").makeLongProcessor(columnValueSelector);
@@ -251,8 +243,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Long> columnValueSelector = TestColumnValueSelector.of(
         Long.class,
-        ImmutableList.of(2L),
-        DateTimes.nowUtc()
+        ImmutableList.of(2L)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("5").makeLongProcessor(columnValueSelector);
@@ -264,8 +255,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        Arrays.asList(null, "v"),
-        DateTimes.nowUtc()
+        Arrays.asList(null, "v")
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector(null).makeComplexProcessor(columnValueSelector);
@@ -277,8 +267,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        Arrays.asList("", "v"),
-        DateTimes.nowUtc()
+        Arrays.asList("", "v")
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector(null).makeComplexProcessor(columnValueSelector);
@@ -294,8 +283,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Integer> columnValueSelector = TestColumnValueSelector.of(
         Integer.class,
-        ImmutableList.of(11),
-        DateTimes.nowUtc()
+        ImmutableList.of(11)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("11").makeComplexProcessor(columnValueSelector);
@@ -307,8 +295,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Integer> columnValueSelector = TestColumnValueSelector.of(
         Integer.class,
-        ImmutableList.of(15),
-        DateTimes.nowUtc()
+        ImmutableList.of(15)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("11").makeComplexProcessor(columnValueSelector);
@@ -320,8 +307,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Long> columnValueSelector = TestColumnValueSelector.of(
         Long.class,
-        ImmutableList.of(11L),
-        DateTimes.nowUtc()
+        ImmutableList.of(11L)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("11").makeComplexProcessor(columnValueSelector);
@@ -333,8 +319,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Long> columnValueSelector = TestColumnValueSelector.of(
         Long.class,
-        ImmutableList.of(15L),
-        DateTimes.nowUtc()
+        ImmutableList.of(15L)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("11").makeComplexProcessor(columnValueSelector);
@@ -346,8 +331,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Float> columnValueSelector = TestColumnValueSelector.of(
         Float.class,
-        ImmutableList.of(11.f),
-        DateTimes.nowUtc()
+        ImmutableList.of(11.f)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("11.f").makeComplexProcessor(columnValueSelector);
@@ -359,8 +343,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Float> columnValueSelector = TestColumnValueSelector.of(
         Float.class,
-        ImmutableList.of(15.f),
-        DateTimes.nowUtc()
+        ImmutableList.of(15.f)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("11.f").makeComplexProcessor(columnValueSelector);
@@ -372,8 +355,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Double> columnValueSelector = TestColumnValueSelector.of(
         Double.class,
-        ImmutableList.of(11.d),
-        DateTimes.nowUtc()
+        ImmutableList.of(11.d)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("11.d").makeComplexProcessor(columnValueSelector);
@@ -385,8 +367,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<Double> columnValueSelector = TestColumnValueSelector.of(
         Double.class,
-        ImmutableList.of(15.d),
-        DateTimes.nowUtc()
+        ImmutableList.of(15.d)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("11.d").makeComplexProcessor(columnValueSelector);
@@ -398,8 +379,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of("val"),
-        DateTimes.nowUtc()
+        ImmutableList.of("val")
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("val").makeComplexProcessor(columnValueSelector);
@@ -411,8 +391,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of("bar"),
-        DateTimes.nowUtc()
+        ImmutableList.of("bar")
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("val").makeComplexProcessor(columnValueSelector);
@@ -424,8 +403,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of(ImmutableList.of("val")),
-        DateTimes.nowUtc()
+        ImmutableList.of(ImmutableList.of("val"))
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("val").makeComplexProcessor(columnValueSelector);
@@ -437,8 +415,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of(ImmutableList.of("bar")),
-        DateTimes.nowUtc()
+        ImmutableList.of(ImmutableList.of("bar"))
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("val").makeComplexProcessor(columnValueSelector);
@@ -450,8 +427,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of(ImmutableList.of()),
-        DateTimes.nowUtc()
+        ImmutableList.of(ImmutableList.of())
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector(null).makeComplexProcessor(columnValueSelector);
@@ -463,8 +439,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of(false),
-        DateTimes.nowUtc()
+        ImmutableList.of(false)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("false").makeComplexProcessor(columnValueSelector);
@@ -476,8 +451,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of(true),
-        DateTimes.nowUtc()
+        ImmutableList.of(true)
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("false").makeComplexProcessor(columnValueSelector);
@@ -489,8 +463,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of(StringUtils.toUtf8("var")),
-        DateTimes.nowUtc()
+        ImmutableList.of(StringUtils.toUtf8("var"))
     );
     columnValueSelector.advance();
     final String base64Encoded = StringUtils.encodeBase64String(StringUtils.toUtf8("var"));
@@ -503,8 +476,7 @@ public class PredicateValueMatcherFactoryTest extends InitializedNullHandlingTes
   {
     final TestColumnValueSelector<String> columnValueSelector = TestColumnValueSelector.of(
         String.class,
-        ImmutableList.of(StringUtils.toUtf8("var")),
-        DateTimes.nowUtc()
+        ImmutableList.of(StringUtils.toUtf8("var"))
     );
     columnValueSelector.advance();
     final ValueMatcher matcher = forSelector("val").makeComplexProcessor(columnValueSelector);
diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
index 09f78e066fb..c82003e1e84 100644
--- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
@@ -50,6 +50,7 @@ import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
+import org.apache.druid.query.groupby.GroupingEngine;
 import org.apache.druid.query.groupby.ResultRow;
 import org.apache.druid.query.groupby.epinephelinae.GroupByQueryEngine;
 import org.apache.druid.query.topn.TopNQueryBuilder;
@@ -58,9 +59,11 @@ import org.apache.druid.query.topn.TopNResultValue;
 import org.apache.druid.segment.CloserRule;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.data.IndexedInts;
 import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.segment.filter.SelectorFilter;
@@ -83,7 +86,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  */
@@ -131,34 +133,34 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
     );
 
 
+    final GroupByQuery query = GroupByQuery.builder()
+                                           .setDataSource("test")
+                                           .setGranularity(Granularities.ALL)
+                                           .setInterval(new Interval(DateTimes.EPOCH, DateTimes.nowUtc()))
+                                           .addDimension("billy")
+                                           .addDimension("sally")
+                                           .addAggregator(new LongSumAggregatorFactory("cnt", "cnt"))
+                                           .addOrderByColumn("billy")
+                                           .build();
+    final CursorBuildSpec buildSpec = GroupingEngine.makeCursorBuildSpec(query, null);
+    final IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index);
     try (
         CloseableStupidPool<ByteBuffer> pool = new CloseableStupidPool<>(
             "GroupByQueryEngine-bufferPool",
             () -> ByteBuffer.allocate(50000)
         );
-        ResourceHolder<ByteBuffer> processingBuffer = pool.take()
+        ResourceHolder<ByteBuffer> processingBuffer = pool.take();
+        final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)
     ) {
-      final GroupByQuery query = GroupByQuery.builder()
-                                             .setDataSource("test")
-                                             .setGranularity(Granularities.ALL)
-                                             .setInterval(new Interval(DateTimes.EPOCH, DateTimes.nowUtc()))
-                                             .addDimension("billy")
-                                             .addDimension("sally")
-                                             .addAggregator(new LongSumAggregatorFactory("cnt", "cnt"))
-                                             .addOrderByColumn("billy")
-                                             .build();
-      final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter()));
-      final Interval interval = Iterables.getOnlyElement(query.getIntervals());
       final Sequence<ResultRow> rows = GroupByQueryEngine.process(
           query,
-          new IncrementalIndexStorageAdapter(index),
+          adapter,
+          cursorHolder,
+          buildSpec,
           processingBuffer.get(),
           null,
           new GroupByQueryConfig(),
-          new DruidProcessingConfig(),
-          filter,
-          interval,
-          null
+          new DruidProcessingConfig()
       );
 
       final List<ResultRow> results = rows.toList();
@@ -195,46 +197,46 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
         )
     );
 
+    final GroupByQuery query = GroupByQuery.builder()
+                                           .setDataSource("test")
+                                           .setGranularity(Granularities.ALL)
+                                           .setInterval(new Interval(DateTimes.EPOCH, DateTimes.nowUtc()))
+                                           .addDimension("billy")
+                                           .addDimension("sally")
+                                           .addAggregator(
+                                               new LongSumAggregatorFactory("cnt", "cnt")
+                                           )
+                                           .addAggregator(
+                                               new JavaScriptAggregatorFactory(
+                                                   "fieldLength",
+                                                   Arrays.asList("sally", "billy"),
+                                                   "function(current, s, b) { return current + (s == null ? 0 : s.length) + (b == null ? 0 : b.length); }",
+                                                   "function() { return 0; }",
+                                                   "function(a,b) { return a + b; }",
+                                                   JavaScriptConfig.getEnabledInstance()
+                                               )
+                                           )
+                                           .addOrderByColumn("billy")
+                                           .build();
+    final IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index);
+    final CursorBuildSpec buildSpec = GroupingEngine.makeCursorBuildSpec(query, null);
     try (
         CloseableStupidPool<ByteBuffer> pool = new CloseableStupidPool<>(
             "GroupByQueryEngine-bufferPool",
             () -> ByteBuffer.allocate(50000)
         );
         ResourceHolder<ByteBuffer> processingBuffer = pool.take();
+        final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)
     ) {
-      final GroupByQuery query = GroupByQuery.builder()
-                                             .setDataSource("test")
-                                             .setGranularity(Granularities.ALL)
-                                             .setInterval(new Interval(DateTimes.EPOCH, DateTimes.nowUtc()))
-                                             .addDimension("billy")
-                                             .addDimension("sally")
-                                             .addAggregator(
-                                                 new LongSumAggregatorFactory("cnt", "cnt")
-                                             )
-                                             .addAggregator(
-                                                 new JavaScriptAggregatorFactory(
-                                                     "fieldLength",
-                                                     Arrays.asList("sally", "billy"),
-                                                     "function(current, s, b) { return current + (s == null ? 0 : s.length) + (b == null ? 0 : b.length); }",
-                                                     "function() { return 0; }",
-                                                     "function(a,b) { return a + b; }",
-                                                     JavaScriptConfig.getEnabledInstance()
-                                                 )
-                                             )
-                                             .addOrderByColumn("billy")
-                                             .build();
-      final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter()));
-      final Interval interval = Iterables.getOnlyElement(query.getIntervals());
       final Sequence<ResultRow> rows = GroupByQueryEngine.process(
           query,
-          new IncrementalIndexStorageAdapter(index),
+          adapter,
+          cursorHolder,
+          buildSpec,
           processingBuffer.get(),
           null,
           new GroupByQueryConfig(),
-          new DruidProcessingConfig(),
-          filter,
-          interval,
-          null
+          new DruidProcessingConfig()
       );
 
       final List<ResultRow> results = rows.toList();
@@ -278,38 +280,38 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
     IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index);
 
     for (boolean descending : Arrays.asList(false, true)) {
-      Sequence<Cursor> cursorSequence = adapter.makeCursors(
-          new SelectorFilter("sally", "bo"),
-          interval,
-          VirtualColumns.EMPTY,
-          Granularities.NONE,
-          descending,
-          null
-      );
+      final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                       .setFilter(new SelectorFilter("sally", "bo"))
+                                                       .setInterval(interval)
+                                                       .setPreferredOrdering(
+                                                           descending ? Cursors.descendingTimeOrder() : null
+                                                       )
+                                                       .build();
+      try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+        Cursor cursor = cursorHolder.asCursor();
+        DimensionSelector dimSelector;
 
-      Cursor cursor = cursorSequence.limit(1).toList().get(0);
-      DimensionSelector dimSelector;
+        dimSelector = cursor
+            .getColumnSelectorFactory()
+            .makeDimensionSelector(new DefaultDimensionSpec("sally", "sally"));
+        Assert.assertEquals("bo", dimSelector.lookupName(dimSelector.getRow().get(0)));
 
-      dimSelector = cursor
-          .getColumnSelectorFactory()
-          .makeDimensionSelector(new DefaultDimensionSpec("sally", "sally"));
-      Assert.assertEquals("bo", dimSelector.lookupName(dimSelector.getRow().get(0)));
+        index.add(
+            new MapBasedInputRow(
+                t.minus(1).getMillis(),
+                Collections.singletonList("sally"),
+                ImmutableMap.of("sally", "ah")
+            )
+        );
 
-      index.add(
-          new MapBasedInputRow(
-              t.minus(1).getMillis(),
-              Collections.singletonList("sally"),
-              ImmutableMap.of("sally", "ah")
-          )
-      );
+        // Cursor reset should not be affected by out of order values
+        cursor.reset();
 
-      // Cursor reset should not be affected by out of order values
-      cursor.reset();
-
-      dimSelector = cursor
-          .getColumnSelectorFactory()
-          .makeDimensionSelector(new DefaultDimensionSpec("sally", "sally"));
-      Assert.assertEquals("bo", dimSelector.lookupName(dimSelector.getRow().get(0)));
+        dimSelector = cursor
+            .getColumnSelectorFactory()
+            .makeDimensionSelector(new DefaultDimensionSpec("sally", "sally"));
+        Assert.assertEquals("bo", dimSelector.lookupName(dimSelector.getRow().get(0)));
+      }
     }
   }
 
@@ -374,36 +376,36 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
         )
     );
 
+
+    final GroupByQuery query = GroupByQuery.builder()
+                                           .setDataSource("test")
+                                           .setGranularity(Granularities.ALL)
+                                           .setInterval(new Interval(DateTimes.EPOCH, DateTimes.nowUtc()))
+                                           .addDimension("billy")
+                                           .addDimension("sally")
+                                           .addAggregator(new LongSumAggregatorFactory("cnt", "cnt"))
+                                           .setDimFilter(DimFilters.dimEquals("sally", (String) null))
+                                           .build();
+    final IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index);
+    final CursorBuildSpec buildSpec = GroupingEngine.makeCursorBuildSpec(query, null);
+
     try (
         CloseableStupidPool<ByteBuffer> pool = new CloseableStupidPool<>(
             "GroupByQueryEngine-bufferPool",
             () -> ByteBuffer.allocate(50000)
         );
         ResourceHolder<ByteBuffer> processingBuffer = pool.take();
+        final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)
     ) {
-
-      final GroupByQuery query = GroupByQuery.builder()
-                                             .setDataSource("test")
-                                             .setGranularity(Granularities.ALL)
-                                             .setInterval(new Interval(DateTimes.EPOCH, DateTimes.nowUtc()))
-                                             .addDimension("billy")
-                                             .addDimension("sally")
-                                             .addAggregator(new LongSumAggregatorFactory("cnt", "cnt"))
-                                             .setDimFilter(DimFilters.dimEquals("sally", (String) null))
-                                             .build();
-      final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter()));
-      final Interval interval = Iterables.getOnlyElement(query.getIntervals());
-
       final Sequence<ResultRow> rows = GroupByQueryEngine.process(
           query,
-          new IncrementalIndexStorageAdapter(index),
+          adapter,
+          cursorHolder,
+          buildSpec,
           processingBuffer.get(),
           null,
           new GroupByQueryConfig(),
-          new DruidProcessingConfig(),
-          filter,
-          interval,
-          null
+          new DruidProcessingConfig()
       );
 
       final List<ResultRow> results = rows.toList();
@@ -433,48 +435,40 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
 
     final StorageAdapter sa = new IncrementalIndexStorageAdapter(index);
 
-    Sequence<Cursor> cursors = sa.makeCursors(
-        null,
-        Intervals.utc(timestamp - 60_000, timestamp + 60_000),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-    final AtomicInteger assertCursorsNotEmpty = new AtomicInteger(0);
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setInterval(Intervals.utc(timestamp - 60_000, timestamp + 60_000))
+                                                     .build();
+    try (final CursorHolder cursorHolder = sa.makeCursorHolder(buildSpec)) {
+      Cursor cursor = cursorHolder.asCursor();
+      DimensionSelector dimSelector = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
+      int cardinality = dimSelector.getValueCardinality();
 
-    cursors
-        .map(cursor -> {
-          DimensionSelector dimSelector = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
-          int cardinality = dimSelector.getValueCardinality();
+      //index gets more rows at this point, while other thread is iterating over the cursor
+      try {
+        for (int i = 0; i < 1; i++) {
+          index.add(new MapBasedInputRow(
+              timestamp,
+              Collections.singletonList("billy"),
+              ImmutableMap.of("billy", "v2" + i)
+          ));
+        }
+      }
+      catch (Exception ex) {
+        throw new RuntimeException(ex);
+      }
 
-          //index gets more rows at this point, while other thread is iterating over the cursor
-          try {
-            for (int i = 0; i < 1; i++) {
-              index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2" + i)));
-            }
-          }
-          catch (Exception ex) {
-            throw new RuntimeException(ex);
-          }
-
-          int rowNumInCursor = 0;
-          // and then, cursoring continues in the other thread
-          while (!cursor.isDone()) {
-            IndexedInts row = dimSelector.getRow();
-            row.forEach(i -> Assert.assertTrue(i < cardinality));
-            cursor.advance();
-            rowNumInCursor++;
-          }
-          Assert.assertEquals(2, rowNumInCursor);
-          assertCursorsNotEmpty.incrementAndGet();
-
-          return null;
-        })
-        .toList();
-    Assert.assertEquals(1, assertCursorsNotEmpty.get());
+      int rowNumInCursor = 0;
+      // and then, cursoring continues in the other thread
+      while (!cursor.isDone()) {
+        IndexedInts row = dimSelector.getRow();
+        row.forEach(i -> Assert.assertTrue(i < cardinality));
+        cursor.advance();
+        rowNumInCursor++;
+      }
+      Assert.assertEquals(2, rowNumInCursor);
+    }
   }
 
   @Test
@@ -497,37 +491,26 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
 
     final StorageAdapter sa = new IncrementalIndexStorageAdapter(index);
 
-    Sequence<Cursor> cursors = sa.makeCursors(
-        new DictionaryRaceTestFilter(index, timestamp),
-        Intervals.utc(timestamp - 60_000, timestamp + 60_000),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-    final AtomicInteger assertCursorsNotEmpty = new AtomicInteger(0);
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(new DictionaryRaceTestFilter(index, timestamp))
+                                                     .setInterval(Intervals.utc(timestamp - 60_000, timestamp + 60_000))
+                                                     .build();
+    try (final CursorHolder cursorHolder = sa.makeCursorHolder(buildSpec)) {
+      Cursor cursor = cursorHolder.asCursor();
+      DimensionSelector dimSelector = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
+      int cardinality = dimSelector.getValueCardinality();
 
-    cursors
-        .map(cursor -> {
-          DimensionSelector dimSelector = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
-          int cardinality = dimSelector.getValueCardinality();
-
-          int rowNumInCursor = 0;
-          while (!cursor.isDone()) {
-            IndexedInts row = dimSelector.getRow();
-            row.forEach(i -> Assert.assertTrue(i < cardinality));
-            cursor.advance();
-            rowNumInCursor++;
-          }
-          Assert.assertEquals(5, rowNumInCursor);
-          assertCursorsNotEmpty.incrementAndGet();
-
-          return null;
-        })
-        .toList();
-    Assert.assertEquals(1, assertCursorsNotEmpty.get());
+      int rowNumInCursor = 0;
+      while (!cursor.isDone()) {
+        IndexedInts row = dimSelector.getRow();
+        row.forEach(i -> Assert.assertTrue(i < cardinality));
+        cursor.advance();
+        rowNumInCursor++;
+      }
+      Assert.assertEquals(5, rowNumInCursor);
+    }
   }
 
   @Test
@@ -548,93 +531,86 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
 
     final StorageAdapter sa = new IncrementalIndexStorageAdapter(index);
 
-    Sequence<Cursor> cursors = sa.makeCursors(
-        null,
-        Intervals.utc(timestamp - 60_000, timestamp + 60_000),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-    final AtomicInteger assertCursorsNotEmpty = new AtomicInteger(0);
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setInterval(Intervals.utc(timestamp - 60_000, timestamp + 60_000))
+                                                     .build();
+    try (final CursorHolder cursorHolder = sa.makeCursorHolder(buildSpec)) {
+      Cursor cursor = cursorHolder.asCursor();
 
-    cursors
-        .map(cursor -> {
-          DimensionSelector dimSelector1A = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
-          int cardinalityA = dimSelector1A.getValueCardinality();
+      DimensionSelector dimSelector1A = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
+      int cardinalityA = dimSelector1A.getValueCardinality();
 
-          //index gets more rows at this point, while other thread is iterating over the cursor
-          try {
-            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v1")));
-          }
-          catch (Exception ex) {
-            throw new RuntimeException(ex);
-          }
+      //index gets more rows at this point, while other thread is iterating over the cursor
+      try {
+        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v1")));
+      }
+      catch (Exception ex) {
+        throw new RuntimeException(ex);
+      }
 
-          DimensionSelector dimSelector1B = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
-          //index gets more rows at this point, while other thread is iterating over the cursor
-          try {
-            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2")));
-            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy2"), ImmutableMap.of("billy2", "v3")));
-          }
-          catch (Exception ex) {
-            throw new RuntimeException(ex);
-          }
+      DimensionSelector dimSelector1B = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
+      //index gets more rows at this point, while other thread is iterating over the cursor
+      try {
+        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2")));
+        index.add(new MapBasedInputRow(
+            timestamp,
+            Collections.singletonList("billy2"),
+            ImmutableMap.of("billy2", "v3")
+        ));
+      }
+      catch (Exception ex) {
+        throw new RuntimeException(ex);
+      }
 
-          DimensionSelector dimSelector1C = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
+      DimensionSelector dimSelector1C = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
 
-          DimensionSelector dimSelector2D = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec("billy2", "billy2"));
-          //index gets more rows at this point, while other thread is iterating over the cursor
-          try {
-            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v3")));
-            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy3"), ImmutableMap.of("billy3", "")));
-          }
-          catch (Exception ex) {
-            throw new RuntimeException(ex);
-          }
+      DimensionSelector dimSelector2D = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec("billy2", "billy2"));
+      //index gets more rows at this point, while other thread is iterating over the cursor
+      try {
+        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v3")));
+        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy3"), ImmutableMap.of("billy3", "")));
+      }
+      catch (Exception ex) {
+        throw new RuntimeException(ex);
+      }
 
-          DimensionSelector dimSelector3E = cursor
-              .getColumnSelectorFactory()
-              .makeDimensionSelector(new DefaultDimensionSpec("billy3", "billy3"));
+      DimensionSelector dimSelector3E = cursor
+          .getColumnSelectorFactory()
+          .makeDimensionSelector(new DefaultDimensionSpec("billy3", "billy3"));
 
-          int rowNumInCursor = 0;
-          // and then, cursoring continues in the other thread
-          while (!cursor.isDone()) {
-            IndexedInts rowA = dimSelector1A.getRow();
-            rowA.forEach(i -> Assert.assertTrue(i < cardinalityA));
-            IndexedInts rowB = dimSelector1B.getRow();
-            rowB.forEach(i -> Assert.assertTrue(i < cardinalityA));
-            IndexedInts rowC = dimSelector1C.getRow();
-            rowC.forEach(i -> Assert.assertTrue(i < cardinalityA));
-            IndexedInts rowD = dimSelector2D.getRow();
-            // no null id, so should get empty dims array
-            Assert.assertEquals(0, rowD.size());
-            IndexedInts rowE = dimSelector3E.getRow();
-            if (NullHandling.replaceWithDefault()) {
-              Assert.assertEquals(1, rowE.size());
-              // the null id
-              Assert.assertEquals(0, rowE.get(0));
-            } else {
-              Assert.assertEquals(0, rowE.size());
-            }
-            cursor.advance();
-            rowNumInCursor++;
-          }
-          Assert.assertEquals(2, rowNumInCursor);
-          assertCursorsNotEmpty.incrementAndGet();
-
-          return null;
-        })
-        .toList();
-    Assert.assertEquals(1, assertCursorsNotEmpty.get());
+      int rowNumInCursor = 0;
+      // and then, cursoring continues in the other thread
+      while (!cursor.isDone()) {
+        IndexedInts rowA = dimSelector1A.getRow();
+        rowA.forEach(i -> Assert.assertTrue(i < cardinalityA));
+        IndexedInts rowB = dimSelector1B.getRow();
+        rowB.forEach(i -> Assert.assertTrue(i < cardinalityA));
+        IndexedInts rowC = dimSelector1C.getRow();
+        rowC.forEach(i -> Assert.assertTrue(i < cardinalityA));
+        IndexedInts rowD = dimSelector2D.getRow();
+        // no null id, so should get empty dims array
+        Assert.assertEquals(0, rowD.size());
+        IndexedInts rowE = dimSelector3E.getRow();
+        if (NullHandling.replaceWithDefault()) {
+          Assert.assertEquals(1, rowE.size());
+          // the null id
+          Assert.assertEquals(0, rowE.get(0));
+        } else {
+          Assert.assertEquals(0, rowE.size());
+        }
+        cursor.advance();
+        rowNumInCursor++;
+      }
+      Assert.assertEquals(2, rowNumInCursor);
+    }
   }
 
   private static class DictionaryRaceTestFilter implements Filter
diff --git a/processing/src/test/java/org/apache/druid/segment/join/BaseHashJoinSegmentStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/join/BaseHashJoinSegmentStorageAdapterTest.java
index e5ab7d1be8a..c3a59659afc 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/BaseHashJoinSegmentStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/BaseHashJoinSegmentStorageAdapterTest.java
@@ -26,6 +26,7 @@ import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.lookup.LookupExtractor;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.QueryableIndexSegment;
 import org.apache.druid.segment.VirtualColumn;
 import org.apache.druid.segment.VirtualColumns;
@@ -222,7 +223,7 @@ public class BaseHashJoinSegmentStorageAdapterTest extends InitializedNullHandli
 
   /**
    * Creates a fact-to-country join segment without a {@link JoinFilterPreAnalysis}. This means it cannot
-   * have {@link org.apache.druid.segment.StorageAdapter#makeCursors} called on it.
+   * have {@link org.apache.druid.segment.StorageAdapter#makeCursorHolder(CursorBuildSpec)} called on it.
    */
   protected HashJoinSegmentStorageAdapter makeFactToCountrySegment()
   {
diff --git a/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
index 5931bff8359..e323b259913 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
@@ -26,13 +26,13 @@ import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.filter.ExpressionDimFilter;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.InDimFilter;
 import org.apache.druid.query.filter.OrDimFilter;
 import org.apache.druid.query.filter.SelectorDimFilter;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ValueType;
@@ -296,7 +296,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeft()
+  public void test_makeCursor_factToCountryLeft()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.LEFT));
 
@@ -305,19 +305,12 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             "countryIsoCode",
@@ -360,7 +353,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeftUsingLookup()
+  public void test_makeCursor_factToCountryLeftUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryNameUsingIsoCodeLookup(JoinType.LEFT));
 
@@ -369,19 +362,12 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             "countryIsoCode",
@@ -423,7 +409,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryInner()
+  public void test_makeCursor_factToCountryInner()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.INNER));
     JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(
@@ -431,19 +417,12 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             "countryIsoCode",
@@ -480,7 +459,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryInnerUsingLookup()
+  public void test_makeCursor_factToCountryInnerUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryNameUsingIsoCodeLookup(JoinType.INNER));
     JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(
@@ -488,19 +467,12 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             "countryIsoCode",
@@ -536,7 +508,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryInnerUsingCountryNumber()
+  public void test_makeCursor_factToCountryInnerUsingCountryNumber()
   {
     // In non-SQL-compatible mode, we get an extra row, since the 'null' countryNumber for "Talk:Oswald Tilghman"
     // is interpreted as 0 (a.k.a. Australia).
@@ -547,18 +519,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -601,7 +568,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryInnerUsingCountryNumberUsingLookup()
+  public void test_makeCursor_factToCountryInnerUsingCountryNumberUsingLookup()
   {
     // In non-SQL-compatible mode, we get an extra row, since the 'null' countryNumber for "Talk:Oswald Tilghman"
     // is interpreted as 0 (a.k.a. Australia).
@@ -612,18 +579,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -664,7 +626,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeftWithFilterOnFacts()
+  public void test_makeCursor_factToCountryLeftWithFilterOnFacts()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.LEFT));
     Filter filter = new SelectorDimFilter("channel", "#de.wikipedia", null).toFilter();
@@ -673,18 +635,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -700,7 +657,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeftWithFilterOnFactsUsingLookup()
+  public void test_makeCursor_factToCountryLeftWithFilterOnFactsUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryNameUsingIsoCodeLookup(JoinType.LEFT));
     Filter filter = new SelectorDimFilter("channel", "#de.wikipedia", null).toFilter();
@@ -709,18 +666,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -735,7 +687,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryRightWithFilterOnLeftIsNull()
+  public void test_makeCursor_factToCountryRightWithFilterOnLeftIsNull()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.RIGHT));
     Filter filter = new SelectorDimFilter("channel", null, null).toFilter();
@@ -744,27 +696,23 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            "countryIsoCode",
-            "countryNumber",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        "countryIsoCode",
+        "countryNumber",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"
+    );
+    CursorBuildSpec buildSpec = CursorBuildSpec.builder().setFilter(filter).build();
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(buildSpec),
+        columns,
         ImmutableList.of(
             new Object[]{null, null, NullHandling.sqlCompatible() ? null : 0L, "AX", "Atlantis", 14L},
             new Object[]{null, null, NullHandling.sqlCompatible() ? null : 0L, "USCA", "Usca", 16L}
@@ -773,7 +721,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryRightWithFilterOnLeftIsNullUsingLookup()
+  public void test_makeCursor_factToCountryRightWithFilterOnLeftIsNullUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryNameUsingIsoCodeLookup(JoinType.RIGHT));
     Filter filter = new SelectorDimFilter("channel", null, null).toFilter();
@@ -782,26 +730,22 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            "countryIsoCode",
-            "countryNumber",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "k",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "v"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        "countryIsoCode",
+        "countryNumber",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "k",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "v"
+    );
+    CursorBuildSpec buildSpec = CursorBuildSpec.builder().setFilter(filter).build();
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(buildSpec),
+        columns,
         ImmutableList.of(
             new Object[]{null, null, NullHandling.sqlCompatible() ? null : 0L, "AX", "Atlantis"},
             new Object[]{null, null, NullHandling.sqlCompatible() ? null : 0L, "USCA", "Usca"}
@@ -810,7 +754,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryFullWithFilterOnLeftIsNull()
+  public void test_makeCursor_factToCountryFullWithFilterOnLeftIsNull()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.FULL));
     Filter filter = new SelectorDimFilter("channel", null, null).toFilter();
@@ -819,27 +763,23 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            "countryIsoCode",
-            "countryNumber",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        "countryIsoCode",
+        "countryNumber",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"
+    );
+    CursorBuildSpec buildSpec = CursorBuildSpec.builder().setFilter(filter).build();
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(buildSpec),
+        columns,
         ImmutableList.of(
             new Object[]{null, null, NullHandling.sqlCompatible() ? null : 0L, "AX", "Atlantis", 14L},
             new Object[]{null, null, NullHandling.sqlCompatible() ? null : 0L, "USCA", "Usca", 16L}
@@ -848,7 +788,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryFullWithFilterOnLeftIsNullUsingLookup()
+  public void test_makeCursor_factToCountryFullWithFilterOnLeftIsNullUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryNameUsingIsoCodeLookup(JoinType.FULL));
     Filter filter = new SelectorDimFilter("channel", null, null).toFilter();
@@ -857,26 +797,22 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            "countryIsoCode",
-            "countryNumber",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "k",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "v"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        "countryIsoCode",
+        "countryNumber",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "k",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "v"
+    );
+    CursorBuildSpec buildSpec = CursorBuildSpec.builder().setFilter(filter).build();
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(buildSpec),
+        columns,
         ImmutableList.of(
             new Object[]{null, null, NullHandling.sqlCompatible() ? null : 0L, "AX", "Atlantis"},
             new Object[]{null, null, NullHandling.sqlCompatible() ? null : 0L, "USCA", "Usca"}
@@ -885,7 +821,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryRightWithFilterOnJoinable()
+  public void test_makeCursor_factToCountryRightWithFilterOnJoinable()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.RIGHT));
     Filter filter = new SelectorDimFilter(
@@ -899,18 +835,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -927,7 +858,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryRightWithFilterOnJoinableUsingLookup()
+  public void test_makeCursor_factToCountryRightWithFilterOnJoinableUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryNameUsingIsoCodeLookup(JoinType.RIGHT));
     Filter filter = new SelectorDimFilter(
@@ -941,18 +872,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -968,7 +894,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeftWithFilterOnJoinable()
+  public void test_makeCursor_factToCountryLeftWithFilterOnJoinable()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.LEFT));
 
@@ -983,18 +909,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1012,7 +933,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeftWithFilterOnJoinableUsingLookup()
+  public void test_makeCursor_factToCountryLeftWithFilterOnJoinableUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryNameUsingIsoCodeLookup(JoinType.LEFT));
     Filter filter = new OrDimFilter(
@@ -1025,18 +946,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1052,7 +968,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryInnerWithFilterInsteadOfRealJoinCondition()
+  public void test_makeCursor_factToCountryInnerWithFilterInsteadOfRealJoinCondition()
   {
     // Join condition => always true.
     // Filter => Fact to countries on countryIsoCode.
@@ -1080,18 +996,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1129,7 +1040,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryInnerWithFilterInsteadOfRealJoinConditionUsingLookup()
+  public void test_makeCursor_factToCountryInnerWithFilterInsteadOfRealJoinConditionUsingLookup()
   {
     // Join condition => always true.
     // Filter => Fact to countries on countryIsoCode.
@@ -1156,18 +1067,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1204,7 +1110,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToRegionToCountryLeft()
+  public void test_makeCursor_factToRegionToCountryLeft()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         factToRegion(JoinType.LEFT),
@@ -1215,19 +1121,12 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             FACT_TO_REGION_PREFIX + "regionName",
@@ -1268,7 +1167,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToRegionToCountryInnerIncludeNull()
+  public void test_makeCursor_factToRegionToCountryInnerIncludeNull()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         factToRegionIncludeNull(JoinType.INNER),
@@ -1279,19 +1178,12 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             FACT_TO_REGION_PREFIX + "regionName",
@@ -1331,7 +1223,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryAlwaysTrue()
+  public void test_makeCursor_factToCountryAlwaysTrue()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         new JoinableClause(
@@ -1352,23 +1244,19 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName"
+    );
+    CursorBuildSpec buildSpec = CursorBuildSpec.builder().setFilter(filter).build();
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(buildSpec),
+        columns,
         ImmutableList.of(
             new Object[]{"Diskussion:Sebastian Schulz", "Australia"},
             new Object[]{"Diskussion:Sebastian Schulz", "Canada"},
@@ -1393,7 +1281,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryAlwaysFalse()
+  public void test_makeCursor_factToCountryAlwaysFalse()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         new JoinableClause(
@@ -1416,18 +1304,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         VirtualColumns.EMPTY
     );
 
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1440,7 +1323,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryAlwaysTrueUsingLookup()
+  public void test_makeCursor_factToCountryAlwaysTrueUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         new JoinableClause(
@@ -1462,23 +1345,19 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "v"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "v"
+    );
+    CursorBuildSpec buildSpec = CursorBuildSpec.builder().setFilter(filter).build();
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(buildSpec),
+        columns,
         ImmutableList.of(
             new Object[]{"Diskussion:Sebastian Schulz", "Australia"},
             new Object[]{"Diskussion:Sebastian Schulz", "Canada"},
@@ -1503,7 +1382,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryAlwaysFalseUsingLookup()
+  public void test_makeCursor_factToCountryAlwaysFalseUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         new JoinableClause(
@@ -1526,18 +1405,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         VirtualColumns.EMPTY
     );
 
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1550,7 +1424,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryUsingVirtualColumn()
+  public void test_makeCursor_factToCountryUsingVirtualColumn()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         new JoinableClause(
@@ -1576,18 +1450,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         virtualColumns
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            virtualColumns,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setVirtualColumns(virtualColumns).build()
         ),
         ImmutableList.of(
             "page",
@@ -1606,7 +1475,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryUsingVirtualColumnUsingLookup()
+  public void test_makeCursor_factToCountryUsingVirtualColumnUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         new JoinableClause(
@@ -1632,18 +1501,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         virtualColumns
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            virtualColumns,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setVirtualColumns(virtualColumns).build()
         ),
         ImmutableList.of(
             "page",
@@ -1662,7 +1526,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryUsingExpression()
+  public void test_makeCursor_factToCountryUsingExpression()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         new JoinableClause(
@@ -1685,19 +1549,12 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             "countryIsoCode",
@@ -1714,7 +1571,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryUsingExpressionUsingLookup()
+  public void test_makeCursor_factToCountryUsingExpressionUsingLookup()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(
         new JoinableClause(
@@ -1737,19 +1594,12 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             "countryIsoCode",
@@ -1766,7 +1616,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToRegionTheWrongWay()
+  public void test_makeCursor_factToRegionTheWrongWay()
   {
     // Joins using only regionIsoCode, which is wrong since they are not unique internationally.
     List<JoinableClause> joinableClauses = ImmutableList.of(
@@ -1791,26 +1641,22 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            "regionIsoCode",
-            "countryIsoCode",
-            FACT_TO_REGION_PREFIX + "regionName",
-            FACT_TO_REGION_PREFIX + "countryIsoCode"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        "regionIsoCode",
+        "countryIsoCode",
+        FACT_TO_REGION_PREFIX + "regionName",
+        FACT_TO_REGION_PREFIX + "countryIsoCode"
+    );
+    CursorBuildSpec buildSpec = CursorBuildSpec.builder().setFilter(filter).build();
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(buildSpec),
+        columns,
         ImmutableList.of(
             new Object[]{"Giusy Ferreri discography", "VA", "IT", "Provincia di Varese", "IT"},
             new Object[]{"Giusy Ferreri discography", "VA", "IT", "Virginia", "US"},
@@ -1823,7 +1669,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_errorOnNonEquiJoin()
+  public void test_makeCursor_errorOnNonEquiJoin()
   {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage("Cannot build hash-join matcher on non-equi-join condition: x == y");
@@ -1847,25 +1693,18 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         VirtualColumns.EMPTY
     );
 
-    JoinTestHelper.readCursors(
+    JoinTestHelper.readCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of()
     );
   }
 
   @Test
-  public void test_makeCursors_errorOnNonEquiJoinUsingLookup()
+  public void test_makeCursor_errorOnNonEquiJoinUsingLookup()
   {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage("Cannot join lookup with non-equi condition: x == y");
@@ -1889,25 +1728,18 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         VirtualColumns.EMPTY
     );
 
-    JoinTestHelper.readCursors(
+    JoinTestHelper.readCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of()
     );
   }
 
   @Test
-  public void test_makeCursors_errorOnNonKeyBasedJoin()
+  public void test_makeCursor_errorOnNonKeyBasedJoin()
   {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage("Cannot build hash-join matcher on non-key-based condition: "
@@ -1931,25 +1763,18 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         VirtualColumns.EMPTY
     );
 
-    JoinTestHelper.readCursors(
+    JoinTestHelper.readCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of()
     );
   }
 
   @Test
-  public void test_makeCursors_errorOnNonKeyBasedJoinUsingLookup()
+  public void test_makeCursor_errorOnNonKeyBasedJoinUsingLookup()
   {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage(
@@ -1973,25 +1798,18 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         VirtualColumns.EMPTY
     );
 
-    JoinTestHelper.readCursors(
+    JoinTestHelper.readCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of()
     );
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeft_filterExcludesAllLeftRows()
+  public void test_makeCursor_factToCountryLeft_filterExcludesAllLeftRows()
   {
     Filter originalFilter = new SelectorFilter("page", "this matches nothing");
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.LEFT));
@@ -2001,18 +1819,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -2026,7 +1839,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeft_filterExcludesAllLeftRowsUsingLookup()
+  public void test_makeCursor_factToCountryLeft_filterExcludesAllLeftRowsUsingLookup()
   {
     Filter originalFilter = new SelectorFilter("page", "this matches nothing");
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryNameUsingIsoCodeLookup(JoinType.LEFT));
@@ -2035,18 +1848,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+        ).makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -2059,7 +1867,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_originalFilterDoesNotMatchPreAnalysis_shouldThrowISE()
+  public void test_makeCursor_originalFilterDoesNotMatchPreAnalysis_shouldThrowISE()
   {
     List<JoinableClause> joinableClauses = ImmutableList.of(factToCountryOnIsoCode(JoinType.LEFT));
     Filter filter = new SelectorFilter("page", "this matches nothing");
@@ -2073,18 +1881,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         factSegment.asStorageAdapter(),
         joinableClauses,
         joinFilterPreAnalysis
-    ).makeCursors(
-        filter,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
+    ).makeCursorHolder(
+        CursorBuildSpec.builder().setFilter(filter).build()
     );
   }
 
   @Test
-  public void test_makeCursors_factToCountryLeftWithBaseFilter()
+  public void test_makeCursor_factToCountryLeftWithBaseFilter()
   {
     final Filter baseFilter = Filters.or(Arrays.asList(
         new SelectorDimFilter("countryIsoCode", "CA", null).toFilter(),
@@ -2098,20 +1901,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             baseFilter,
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             "countryIsoCode",
@@ -2129,7 +1925,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryInnerWithBaseFilter()
+  public void test_makeCursor_factToCountryInnerWithBaseFilter()
   {
     final Filter baseFilter = Filters.or(Arrays.asList(
         new SelectorDimFilter("countryIsoCode", "CA", null).toFilter(),
@@ -2142,20 +1938,13 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
+    JoinTestHelper.verifyCursor(
         new HashJoinSegmentStorageAdapter(
             factSegment.asStorageAdapter(),
             baseFilter,
             joinableClauses,
             joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
+        ).makeCursorHolder(CursorBuildSpec.FULL_SCAN),
         ImmutableList.of(
             "page",
             "countryIsoCode",
@@ -2172,7 +1961,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryRightWithBaseFilter()
+  public void test_makeCursor_factToCountryRightWithBaseFilter()
   {
     final Filter baseFilter = Filters.or(Arrays.asList(
         new SelectorDimFilter("countryIsoCode", "CA", null).toFilter(),
@@ -2185,27 +1974,22 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            baseFilter,
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            "countryIsoCode",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        baseFilter,
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        "countryIsoCode",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"
+    );
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN),
+        columns,
         ImmutableList.of(
             new Object[]{"Didier Leclair", "CA", "CA", "Canada", 1L},
             new Object[]{"Les Argonautes", "CA", "CA", "Canada", 1L},
@@ -2232,7 +2016,7 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
   }
 
   @Test
-  public void test_makeCursors_factToCountryFullWithBaseFilter()
+  public void test_makeCursor_factToCountryFullWithBaseFilter()
   {
     final Filter baseFilter = Filters.or(Arrays.asList(
         new SelectorDimFilter("countryIsoCode", "CA", null).toFilter(),
@@ -2245,27 +2029,22 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
         joinableClauses,
         VirtualColumns.EMPTY
     );
-    JoinTestHelper.verifyCursors(
-        new HashJoinSegmentStorageAdapter(
-            factSegment.asStorageAdapter(),
-            baseFilter,
-            joinableClauses,
-            joinFilterPreAnalysis
-        ).makeCursors(
-            null,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
-        ),
-        ImmutableList.of(
-            "page",
-            "countryIsoCode",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName",
-            FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"
-        ),
+    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(
+        factSegment.asStorageAdapter(),
+        baseFilter,
+        joinableClauses,
+        joinFilterPreAnalysis
+    );
+    List<String> columns = ImmutableList.of(
+        "page",
+        "countryIsoCode",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName",
+        FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"
+    );
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN),
+        columns,
         ImmutableList.of(
             new Object[]{"Didier Leclair", "CA", "CA", "Canada", 1L},
             new Object[]{"Les Argonautes", "CA", "CA", "Canada", 1L},
diff --git a/processing/src/test/java/org/apache/druid/segment/join/JoinFilterAnalyzerTest.java b/processing/src/test/java/org/apache/druid/segment/join/JoinFilterAnalyzerTest.java
index 68a77f38884..656cbdb2ba2 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/JoinFilterAnalyzerTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/JoinFilterAnalyzerTest.java
@@ -24,9 +24,7 @@ import com.google.common.collect.ImmutableSet;
 import nl.jqno.equalsverifier.EqualsVerifier;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.expression.TestExprMacroTable;
@@ -34,6 +32,7 @@ import org.apache.druid.query.filter.BoundDimFilter;
 import org.apache.druid.query.filter.ExpressionDimFilter;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.InDimFilter;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.VirtualColumn;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnType;
@@ -82,14 +81,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -159,14 +153,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -215,14 +204,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -282,14 +266,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -353,14 +332,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -423,14 +397,12 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            virtualColumns,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder()
+                           .setFilter(originalFilter)
+                           .setVirtualColumns(virtualColumns)
+                           .build()
         ),
         ImmutableList.of(
             "page",
@@ -497,14 +469,12 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            virtualColumns,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder()
+                           .setFilter(originalFilter)
+                           .setVirtualColumns(virtualColumns)
+                           .build()
         ),
         ImmutableList.of(
             "page",
@@ -601,14 +571,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -728,14 +693,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -822,14 +782,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinableClauses,
         joinFilterPreAnalysis
     );
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -872,14 +827,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -974,14 +924,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1064,14 +1009,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1138,14 +1078,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1196,14 +1131,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1253,14 +1183,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1309,14 +1234,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1366,14 +1286,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1423,14 +1338,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
 
     // In non-SQL-compatible mode, we get an extra row, since the 'null' countryNumber for "Talk:Oswald Tilghman"
     // is interpreted as 0 (a.k.a. Australia).
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1487,14 +1397,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
 
     // In non-SQL-compatible mode, we get an extra row, since the 'null' countryNumber for "Talk:Oswald Tilghman"
     // is interpreted as 0 (a.k.a. Australia).
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1548,14 +1453,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1603,14 +1503,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1657,14 +1552,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1715,14 +1605,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            filter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(filter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1772,14 +1657,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1828,14 +1708,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1895,14 +1770,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -1966,14 +1836,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -2047,14 +1912,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         + "Equality{leftExpr=user, rightColumn='regionName', includeNull=false}"
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -2108,14 +1968,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -2188,14 +2043,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -2391,14 +2241,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
       expectedVirtualColumns = ImmutableSet.of();
     }
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -2525,14 +2370,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
@@ -2605,14 +2445,9 @@ public class JoinFilterAnalyzerTest extends BaseHashJoinSegmentStorageAdapterTes
         joinFilterPreAnalysis
     );
 
-    JoinTestHelper.verifyCursors(
-        adapter.makeCursors(
-            originalFilter,
-            Intervals.ETERNITY,
-            VirtualColumns.EMPTY,
-            Granularities.ALL,
-            false,
-            null
+    JoinTestHelper.verifyCursor(
+        adapter.makeCursorHolder(
+            CursorBuildSpec.builder().setFilter(originalFilter).build()
         ),
         ImmutableList.of(
             "page",
diff --git a/processing/src/test/java/org/apache/druid/segment/join/JoinTestHelper.java b/processing/src/test/java/org/apache/druid/segment/join/JoinTestHelper.java
index 86d614cf4ac..33ad0dfb6a8 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/JoinTestHelper.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/JoinTestHelper.java
@@ -34,8 +34,6 @@ import org.apache.druid.data.input.impl.StringDimensionSchema;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.jackson.JacksonUtils;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory;
@@ -48,6 +46,7 @@ import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnProcessorFactory;
 import org.apache.druid.segment.ColumnProcessors;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionHandlerUtils;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IndexBuilder;
@@ -325,54 +324,70 @@ public class JoinTestHelper
     );
   }
 
-  public static List<Object[]> readCursors(final Sequence<Cursor> cursors, final List<String> columns)
+  public static List<Object[]> readCursor(final CursorHolder cursorHolder, final List<String> columns)
   {
-    return cursors.flatMap(
-        cursor -> {
-          final List<Supplier<Object>> readers = columns
-              .stream()
-              .map(
-                  column ->
-                      ColumnProcessors.makeProcessor(
-                          column,
-                          SIMPLE_READER,
-                          cursor.getColumnSelectorFactory()
-                      )
-              )
-              .collect(Collectors.toList());
+    try {
+      final Cursor cursor = cursorHolder.asCursor();
+      final List<Supplier<Object>> readers = columns
+          .stream()
+          .map(
+              column ->
+                  ColumnProcessors.makeProcessor(
+                      column,
+                      SIMPLE_READER,
+                      cursor.getColumnSelectorFactory()
+                  )
+          )
+          .collect(Collectors.toList());
 
-          final List<Object[]> rows = new ArrayList<>();
-          boolean interruptible = false; // test both advance() and advanceUninterruptibly()
+      final List<Object[]> rows = new ArrayList<>();
+      boolean interruptible = false; // test both advance() and advanceUninterruptibly()
 
-          while (!cursor.isDone()) {
-            final Object[] row = new Object[columns.size()];
-
-            for (int i = 0; i < row.length; i++) {
-              row[i] = readers.get(i).get();
-            }
-
-            rows.add(row);
-            if (interruptible) {
-              cursor.advance();
-            } else {
-              cursor.advanceUninterruptibly();
-            }
-
-            interruptible = !interruptible;
-          }
-
-          return Sequences.simple(rows);
+      // test cursor reset
+      while (!cursor.isDone()) {
+        if (interruptible) {
+          cursor.advance();
+        } else {
+          cursor.advanceUninterruptibly();
         }
-    ).toList();
+
+        interruptible = !interruptible;
+      }
+
+      cursor.reset();
+
+      while (!cursor.isDone()) {
+        final Object[] row = new Object[columns.size()];
+
+        for (int i = 0; i < row.length; i++) {
+          row[i] = readers.get(i).get();
+        }
+
+        rows.add(row);
+        if (interruptible) {
+          cursor.advance();
+        } else {
+          cursor.advanceUninterruptibly();
+        }
+
+        interruptible = !interruptible;
+      }
+
+      return rows;
+    }
+    finally {
+      cursorHolder.close();
+    }
   }
 
-  public static void verifyCursors(
-      final Sequence<Cursor> cursors,
+
+  public static void verifyCursor(
+      final CursorHolder cursorHolder,
       final List<String> columns,
       final List<Object[]> expectedRows
   )
   {
-    final List<Object[]> rows = readCursors(cursors, columns);
+    final List<Object[]> rows = readCursor(cursorHolder, columns);
 
     for (int i = 0; i < rows.size(); i++) {
       try {
diff --git a/processing/src/test/java/org/apache/druid/segment/join/PostJoinCursorTest.java b/processing/src/test/java/org/apache/druid/segment/join/PostJoinCursorTest.java
index 6813c04bb10..1a5e6dc63de 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/PostJoinCursorTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/PostJoinCursorTest.java
@@ -20,18 +20,14 @@
 package org.apache.druid.segment.join;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.QueryInterruptedException;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexSegment;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
@@ -39,8 +35,6 @@ import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.join.filter.JoinFilterPreAnalysis;
 import org.apache.druid.timeline.SegmentId;
-import org.joda.time.DateTime;
-import org.joda.time.Interval;
 import org.junit.Test;
 
 import javax.annotation.Nullable;
@@ -76,17 +70,31 @@ public class PostJoinCursorTest extends BaseHashJoinSegmentStorageAdapterTest
       }
 
       @Override
-      public Sequence<Cursor> makeCursors(
-          @Nullable Filter filter,
-          Interval interval,
-          VirtualColumns virtualColumns,
-          Granularity gran,
-          boolean descending,
-          @Nullable QueryMetrics<?> queryMetrics
-      )
+      public CursorHolder makeCursorHolder(CursorBuildSpec spec)
       {
-        return super.makeCursors(filter, interval, virtualColumns, gran, descending, queryMetrics)
-                    .map(cursor -> new CursorNoAdvance(cursor, countDownLatch));
+        final CursorHolder delegate = super.makeCursorHolder(spec);
+        return new CursorHolder()
+        {
+          @Nullable
+          @Override
+          public Cursor asCursor()
+          {
+            return new CursorNoAdvance(delegate.asCursor(), countDownLatch);
+          }
+
+          @Nullable
+          @Override
+          public List<OrderBy> getOrdering()
+          {
+            return delegate.getOrdering();
+          }
+
+          @Override
+          public void close()
+          {
+            delegate.close();
+          }
+        };
       }
 
       private static class CursorNoAdvance implements Cursor
@@ -106,12 +114,6 @@ public class PostJoinCursorTest extends BaseHashJoinSegmentStorageAdapterTest
           return cursor.getColumnSelectorFactory();
         }
 
-        @Override
-        public DateTime getTime()
-        {
-          return cursor.getTime();
-        }
-
         @Override
         public void advance()
         {
@@ -142,7 +144,7 @@ public class PostJoinCursorTest extends BaseHashJoinSegmentStorageAdapterTest
         @Override
         public void reset()
         {
-
+          cursor.reset();
         }
       }
     }
@@ -235,30 +237,25 @@ public class PostJoinCursorTest extends BaseHashJoinSegmentStorageAdapterTest
         joinFilterPreAnalysis
     );
 
-    Cursor cursor = Iterables.getOnlyElement(hashJoinSegmentStorageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    ).toList());
+    try (final CursorHolder cursorHolder = hashJoinSegmentStorageAdapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
 
-    ((PostJoinCursor) cursor).setValueMatcher(new ValueMatcher()
-    {
-      @Override
-      public boolean matches(boolean includeUnknown)
+      ((PostJoinCursor) cursor).setValueMatcher(new ValueMatcher()
       {
-        return false;
-      }
+        @Override
+        public boolean matches(boolean includeUnknown)
+        {
+          return false;
+        }
 
-      @Override
-      public void inspectRuntimeShape(RuntimeShapeInspector inspector)
-      {
+        @Override
+        public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+        {
 
-      }
-    });
+        }
+      });
 
-    cursor.advance();
+      cursor.advance();
+    }
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java
index e46b7893a0d..fd89e083441 100644
--- a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java
@@ -23,11 +23,7 @@ import com.fasterxml.jackson.databind.Module;
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.error.DruidException;
 import org.apache.druid.guice.BuiltInTypesModule;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Yielder;
-import org.apache.druid.java.util.common.guava.Yielders;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.query.NestedDataTestUtils;
 import org.apache.druid.query.aggregation.AggregationTestHelper;
@@ -35,6 +31,8 @@ import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DoubleColumnSelector;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.LongColumnSelector;
@@ -347,17 +345,11 @@ public class NestedFieldColumnSelectorsTest extends InitializedNullHandlingTest
     );
     Assert.assertEquals(1, segments.size());
     StorageAdapter storageAdapter = segments.get(0).asStorageAdapter();
-    Sequence<Cursor> cursorSequence = storageAdapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        virtualColumns,
-        Granularities.DAY,
-        false,
-        null
-    );
-    final Yielder<Cursor> yielder = Yielders.each(cursorSequence);
-    closer.register(yielder);
-    final Cursor cursor = yielder.get();
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(virtualColumns)
+                                                     .build();
+    final CursorHolder cursorHolder = closer.register(storageAdapter.makeCursorHolder(buildSpec));
+    final Cursor cursor = cursorHolder.asCursor();
     return cursor.getColumnSelectorFactory();
   }
 
@@ -378,14 +370,8 @@ public class NestedFieldColumnSelectorsTest extends InitializedNullHandlingTest
     );
     Assert.assertEquals(1, segments.size());
     StorageAdapter storageAdapter = segments.get(0).asStorageAdapter();
-    VectorCursor cursor = storageAdapter.makeVectorCursor(
-        null,
-        Intervals.ETERNITY,
-        virtualColumns,
-        false,
-        512,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder().setVirtualColumns(virtualColumns).build();
+    VectorCursor cursor = closer.register(storageAdapter.makeCursorHolder(buildSpec)).asVectorCursor();
     return cursor.getColumnSelectorFactory();
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/selector/TestColumnValueSelector.java b/processing/src/test/java/org/apache/druid/segment/selector/TestColumnValueSelector.java
index 8e3ad9c64d4..09f51634bea 100644
--- a/processing/src/test/java/org/apache/druid/segment/selector/TestColumnValueSelector.java
+++ b/processing/src/test/java/org/apache/druid/segment/selector/TestColumnValueSelector.java
@@ -26,7 +26,6 @@ import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.column.ColumnCapabilities;
-import org.joda.time.DateTime;
 
 import javax.annotation.Nullable;
 import java.util.Collection;
@@ -38,26 +37,24 @@ public class TestColumnValueSelector<T> implements ColumnValueSelector<Object>,
 {
   private final Class<T> clazz;
   private final Supplier<Iterator<Object>> iteratorSupplier;
-  private final DateTime time;
 
   private Iterator<Object> iterator;
   private Object value;
 
-  public static <T> TestColumnValueSelector<T> of(Class<T> clazz, Collection<Object> collection, DateTime time)
+  public static <T> TestColumnValueSelector<T> of(Class<T> clazz, Collection<Object> collection)
   {
-    return new TestColumnValueSelector<>(clazz, collection::iterator, time);
+    return new TestColumnValueSelector<>(clazz, collection::iterator);
   }
 
-  public static <T> TestColumnValueSelector<T> of(Class<T> clazz, Stream<Object> stream, DateTime time)
+  public static <T> TestColumnValueSelector<T> of(Class<T> clazz, Stream<Object> stream)
   {
-    return new TestColumnValueSelector<>(clazz, stream::iterator, time);
+    return new TestColumnValueSelector<>(clazz, stream::iterator);
   }
 
-  protected TestColumnValueSelector(Class<T> clazz, Supplier<Iterator<Object>> iteratorSupplier, DateTime time)
+  protected TestColumnValueSelector(Class<T> clazz, Supplier<Iterator<Object>> iteratorSupplier)
   {
     this.clazz = clazz;
     this.iteratorSupplier = iteratorSupplier;
-    this.time = time;
     this.iterator = iteratorSupplier.get();
   }
 
@@ -87,12 +84,6 @@ public class TestColumnValueSelector<T> implements ColumnValueSelector<Object>,
     };
   }
 
-  @Override
-  public DateTime getTime()
-  {
-    return time;
-  }
-
   @Override
   public void advance()
   {
diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
index b6b9713682a..13f82e2be8e 100644
--- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
@@ -28,9 +28,7 @@ import org.apache.druid.data.input.MapBasedInputRow;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprEval;
@@ -45,6 +43,8 @@ import org.apache.druid.segment.BaseSingleValueDimensionSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
@@ -138,17 +138,9 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
   {
     final String columnName = "string3";
     for (StorageAdapter adapter : ADAPTERS) {
-      Sequence<Cursor> cursorSequence = adapter.makeCursors(
-          null,
-          adapter.getInterval(),
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
+      try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        Cursor cursor = cursorHolder.asCursor();
 
-
-      cursorSequence.accumulate(null, (accumulated, cursor) -> {
         ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
         ExpressionPlan plan = ExpressionPlanner.plan(
             adapter,
@@ -202,9 +194,7 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
 
           cursor.advance();
         }
-
-        return null;
-      });
+      }
     }
   }
 
@@ -213,16 +203,8 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
   {
     final String columnName = "multi-string3";
     for (StorageAdapter adapter : ADAPTERS) {
-      Sequence<Cursor> cursorSequence = adapter.makeCursors(
-          null,
-          adapter.getInterval(),
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
-
-      cursorSequence.accumulate(null, (ignored, cursor) -> {
+      try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        Cursor cursor = cursorHolder.asCursor();
         ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
 
         // identifier, uses dimension selector supplier supplier, no null coercion
@@ -289,8 +271,7 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
 
           cursor.advance();
         }
-        return ignored;
-      });
+      }
     }
   }
 
@@ -299,16 +280,8 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
   {
     final String columnName = "long3";
     for (StorageAdapter adapter : ADAPTERS) {
-      Sequence<Cursor> cursorSequence = adapter.makeCursors(
-          null,
-          adapter.getInterval(),
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
-
-      cursorSequence.accumulate(null, (accumulated, cursor) -> {
+      try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        Cursor cursor = cursorHolder.asCursor();
         ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
         // an assortment of plans
         ExpressionPlan plan = ExpressionPlanner.plan(
@@ -343,9 +316,7 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
           }
           cursor.advance();
         }
-
-        return null;
-      });
+      }
     }
   }
 
@@ -354,17 +325,8 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
   {
     final String columnName = "double3";
     for (StorageAdapter adapter : ADAPTERS) {
-      Sequence<Cursor> cursorSequence = adapter.makeCursors(
-          null,
-          adapter.getInterval(),
-          VirtualColumns.EMPTY,
-          Granularities.ALL,
-          false,
-          null
-      );
-
-
-      cursorSequence.accumulate(null, (accumulated, cursor) -> {
+      try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+        Cursor cursor = cursorHolder.asCursor();
         ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
         // an assortment of plans
         ExpressionPlan plan = ExpressionPlanner.plan(
@@ -399,9 +361,7 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
           }
           cursor.advance();
         }
-
-        return null;
-      });
+      }
     }
   }
 
@@ -684,16 +644,8 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
     );
 
     IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index);
-
-    Sequence<Cursor> cursors = adapter.makeCursors(
-        null,
-        Intervals.ETERNITY,
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
-    int rowsProcessed = cursors.map(cursor -> {
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Cursor cursor = cursorHolder.asCursor();
       DimensionSelector xExprSelector = ExpressionSelectors.makeDimensionSelector(
           cursor.getColumnSelectorFactory(),
           Parser.parse("concat(x, 'foo')", ExprMacroTable.nil()),
@@ -722,10 +674,9 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
         rowCount++;
         cursor.advance();
       }
-      return rowCount;
-    }).accumulate(0, (in, acc) -> in + acc);
 
-    Assert.assertEquals(2, rowsProcessed);
+      Assert.assertEquals(2, rowCount);
+    }
   }
 
   private static DimensionSelector dimensionSelectorFromSupplier(
diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsCastTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsCastTest.java
index 6f733ee0b91..6fbee67734f 100644
--- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsCastTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsCastTest.java
@@ -24,6 +24,7 @@ import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.ExpressionType;
 import org.apache.druid.query.expression.TestExprMacroTable;
+import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
@@ -116,15 +117,10 @@ public class ExpressionVectorSelectorsCastTest
         )
     );
     final QueryableIndexStorageAdapter storageAdapter = new QueryableIndexStorageAdapter(index);
-    VectorCursor cursor = storageAdapter.makeVectorCursor(
-        null,
-        index.getDataInterval(),
-        virtualColumns,
-        false,
-        512,
-        null
-    );
-    closer.register(cursor);
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(virtualColumns)
+                                                     .build();
+    VectorCursor cursor = closer.register(storageAdapter.makeCursorHolder(buildSpec)).asVectorCursor();
 
     ColumnCapabilities capabilities = INDEX.getColumnCapabilities(column);
 
diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java
index 6d27439f8a6..5d284a74129 100644
--- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java
@@ -22,7 +22,6 @@ package org.apache.druid.segment.virtual;
 import com.google.common.collect.ImmutableList;
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
@@ -36,6 +35,8 @@ import org.apache.druid.query.groupby.epinephelinae.collection.MemoryPointer;
 import org.apache.druid.query.groupby.epinephelinae.vector.GroupByVectorColumnSelector;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DeprecatedQueryableIndexColumnSelector;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.QueryableIndex;
@@ -234,109 +235,101 @@ public class ExpressionVectorSelectorsTest extends InitializedNullHandlingTest
         )
     );
     final QueryableIndexStorageAdapter storageAdapter = new QueryableIndexStorageAdapter(index);
-    VectorCursor cursor = storageAdapter.makeVectorCursor(
-        null,
-        index.getDataInterval(),
-        virtualColumns,
-        false,
-        512,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setVirtualColumns(virtualColumns)
+                                                     .build();
+    try (final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(buildSpec)) {
+      final VectorCursor cursor = cursorHolder.asVectorCursor();
+      Assert.assertNotNull(cursor);
 
-    ColumnCapabilities capabilities = virtualColumns.getColumnCapabilitiesWithFallback(storageAdapter, "v");
+      ColumnCapabilities capabilities = virtualColumns.getColumnCapabilitiesWithFallback(storageAdapter, "v");
 
-    int rowCount = 0;
-    if (capabilities.isDictionaryEncoded().isTrue()) {
-      SingleValueDimensionVectorSelector selector = cursor.getColumnSelectorFactory().makeSingleValueDimensionSelector(
-          DefaultDimensionSpec.of("v")
-      );
-      while (!cursor.isDone()) {
-        int[] row = selector.getRowVector();
-        for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
-          results.add(selector.lookupName(row[i]));
-        }
-        cursor.advance();
-      }
-    } else {
-      VectorValueSelector selector = null;
-      VectorObjectSelector objectSelector = null;
-      if (Types.isNumeric(outputType)) {
-        selector = cursor.getColumnSelectorFactory().makeValueSelector("v");
-      } else {
-        objectSelector = cursor.getColumnSelectorFactory().makeObjectSelector("v");
-      }
-      GroupByVectorColumnSelector groupBySelector =
-          cursor.getColumnSelectorFactory().makeGroupByVectorColumnSelector("v", DeferExpressionDimensions.ALWAYS);
-      while (!cursor.isDone()) {
-        final List<Object> resultsVector = new ArrayList<>();
-        boolean[] nulls;
-        switch (outputType.getType()) {
-          case LONG:
-            nulls = selector.getNullVector();
-            long[] longs = selector.getLongVector();
-            for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
-              resultsVector.add(nulls != null && nulls[i] ? null : longs[i]);
-            }
-            break;
-          case DOUBLE:
-            // special case to test floats just to get coverage on getFloatVector
-            if ("float2".equals(expression)) {
-              nulls = selector.getNullVector();
-              float[] floats = selector.getFloatVector();
-              for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
-                resultsVector.add(nulls != null && nulls[i] ? null : (double) floats[i]);
-              }
-            } else {
-              nulls = selector.getNullVector();
-              double[] doubles = selector.getDoubleVector();
-              for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
-                resultsVector.add(nulls != null && nulls[i] ? null : doubles[i]);
-              }
-            }
-            break;
-          case STRING:
-            Object[] objects = objectSelector.getObjectVector();
-            for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++, rowCount++) {
-              resultsVector.add(objects[i]);
-            }
-            break;
-        }
-
-        verifyGroupBySelector(groupBySelector, resultsVector);
-        results.addAll(resultsVector);
-        cursor.advance();
-      }
-    }
-    closer.register(cursor);
-
-    Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(
-        null,
-        index.getDataInterval(),
-        virtualColumns,
-        Granularities.ALL,
-        false,
-        null
-    );
-
-    int rowCountCursor = cursors
-        .map(nonVectorized -> {
-          final ColumnValueSelector nonSelector = nonVectorized.getColumnSelectorFactory()
-                                                               .makeColumnValueSelector("v");
-          int rows = 0;
-          while (!nonVectorized.isDone()) {
-            Assert.assertEquals(
-                "Failed at row " + rows,
-                nonSelector.getObject(),
-                results.get(rows)
-            );
-            rows++;
-            nonVectorized.advance();
+      int rowCount = 0;
+      if (capabilities.isDictionaryEncoded().isTrue()) {
+        SingleValueDimensionVectorSelector selector = cursor.getColumnSelectorFactory()
+                                                            .makeSingleValueDimensionSelector(
+                                                                DefaultDimensionSpec.of("v")
+                                                            );
+        while (!cursor.isDone()) {
+          int[] row = selector.getRowVector();
+          for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
+            results.add(selector.lookupName(row[i]));
+          }
+          cursor.advance();
+        }
+      } else {
+        VectorValueSelector selector = null;
+        VectorObjectSelector objectSelector = null;
+        if (Types.isNumeric(outputType)) {
+          selector = cursor.getColumnSelectorFactory().makeValueSelector("v");
+        } else {
+          objectSelector = cursor.getColumnSelectorFactory().makeObjectSelector("v");
+        }
+        GroupByVectorColumnSelector groupBySelector =
+            cursor.getColumnSelectorFactory().makeGroupByVectorColumnSelector("v", DeferExpressionDimensions.ALWAYS);
+        while (!cursor.isDone()) {
+          final List<Object> resultsVector = new ArrayList<>();
+          boolean[] nulls;
+          switch (outputType.getType()) {
+            case LONG:
+              Assert.assertNotNull(selector);
+              nulls = selector.getNullVector();
+              long[] longs = selector.getLongVector();
+              for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
+                resultsVector.add(nulls != null && nulls[i] ? null : longs[i]);
+              }
+              break;
+            case DOUBLE:
+              Assert.assertNotNull(selector);
+              // special case to test floats just to get coverage on getFloatVector
+              if ("float2".equals(expression)) {
+                nulls = selector.getNullVector();
+                float[] floats = selector.getFloatVector();
+                for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
+                  resultsVector.add(nulls != null && nulls[i] ? null : (double) floats[i]);
+                }
+              } else {
+                nulls = selector.getNullVector();
+                double[] doubles = selector.getDoubleVector();
+                for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
+                  resultsVector.add(nulls != null && nulls[i] ? null : doubles[i]);
+                }
+              }
+              break;
+            default:
+              Assert.assertNotNull(objectSelector);
+              Object[] objects = objectSelector.getObjectVector();
+              for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++, rowCount++) {
+                resultsVector.add(objects[i]);
+              }
+              break;
           }
-          return rows;
-        }).accumulate(0, (acc, in) -> acc + in);
 
-    Assert.assertTrue(rowCountCursor > 0);
-    Assert.assertEquals(rowCountCursor, rowCount);
+          verifyGroupBySelector(groupBySelector, resultsVector);
+          results.addAll(resultsVector);
+          cursor.advance();
+        }
+      }
+
+
+      final Cursor nonVectorized = cursorHolder.asCursor();
+      Assert.assertNotNull(nonVectorized);
+      final ColumnValueSelector nonSelector = nonVectorized.getColumnSelectorFactory()
+                                                           .makeColumnValueSelector("v");
+      int rows = 0;
+      while (!nonVectorized.isDone()) {
+        Assert.assertEquals(
+            "Failed at row " + rows,
+            nonSelector.getObject(),
+            results.get(rows)
+        );
+        rows++;
+        nonVectorized.advance();
+      }
+
+      Assert.assertTrue(rows > 0);
+      Assert.assertEquals(rows, rowCount);
+    }
   }
 
   private static void verifyGroupBySelector(
diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java
index 6270f368540..b31799cf8b4 100644
--- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java
+++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java
@@ -155,7 +155,6 @@ public class CachingClusteredClientPerfTest
     return new TestQuery(
         new TableDataSource("test"),
         new MultipleIntervalSegmentSpec(Collections.singletonList(interval)),
-        false,
         ImmutableMap.of(BaseQuery.QUERY_ID, "testQuery")
     );
   }
@@ -202,11 +201,10 @@ public class CachingClusteredClientPerfTest
     public TestQuery(
         DataSource dataSource,
         QuerySegmentSpec querySegmentSpec,
-        boolean descending,
         Map<String, Object> context
     )
     {
-      super(dataSource, querySegmentSpec, descending, context);
+      super(dataSource, querySegmentSpec, context);
     }
 
     @Override
diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTest.java
index 538784a8853..46580a5eeee 100644
--- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTest.java
+++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTest.java
@@ -35,6 +35,7 @@ import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.query.Druids;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.SegmentDescriptor;
@@ -1421,7 +1422,7 @@ public class StreamAppenderatorTest extends InitializedNullHandlingTest
                                              )
                                          )
                                      )
-                                     .order(ScanQuery.Order.ASCENDING)
+                                     .order(Order.ASCENDING)
                                      .batchSize(10)
                                      .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                                      .build();
diff --git a/server/src/test/java/org/apache/druid/server/TestSegmentUtils.java b/server/src/test/java/org/apache/druid/server/TestSegmentUtils.java
index c58c686074d..14eb3d44220 100644
--- a/server/src/test/java/org/apache/druid/server/TestSegmentUtils.java
+++ b/server/src/test/java/org/apache/druid/server/TestSegmentUtils.java
@@ -27,11 +27,9 @@ import com.google.common.io.Files;
 import org.apache.druid.collections.bitmap.BitmapFactory;
 import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionHandler;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.Metadata;
@@ -39,7 +37,6 @@ import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.SegmentLazyLoadFailCallback;
 import org.apache.druid.segment.StorageAdapter;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.data.Indexed;
@@ -332,16 +329,17 @@ public class TestSegmentUtils
         }
 
         @Override
-        public Sequence<Cursor> makeCursors(
-            @Nullable Filter filter,
-            Interval interval,
-            VirtualColumns virtualColumns,
-            Granularity gran,
-            boolean descending,
-            @Nullable QueryMetrics<?> queryMetrics
-        )
+        public CursorHolder makeCursorHolder(CursorBuildSpec spec)
         {
-          return null;
+          return new CursorHolder()
+          {
+            @Nullable
+            @Override
+            public Cursor asCursor()
+            {
+              return null;
+            }
+          };
         }
       };
 
diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java
index 9f20dc3be31..ec4a008c1ed 100644
--- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java
@@ -599,7 +599,6 @@ public class ServerManagerTest
         new BaseQuery<Object>(
             new TableDataSource("test"),
             new MultipleSpecificSegmentSpec(descriptors),
-            false,
             new HashMap<>()
         )
         {
diff --git a/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java b/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java
index f3dd1eb6809..0ee15ab1f3f 100644
--- a/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java
+++ b/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java
@@ -87,7 +87,8 @@ public class LoggingRequestLoggerTest
         {
           return null;
         }
-      }, false, queryContext
+      },
+      queryContext
   );
 
   final Query<?> nestedQuery = new FakeQuery(
@@ -105,7 +106,8 @@ public class LoggingRequestLoggerTest
         {
           return null;
         }
-      }, false, queryContext
+      },
+      queryContext
   );
 
   final Query nestedNestedQuery = new FakeQuery(
@@ -123,7 +125,8 @@ public class LoggingRequestLoggerTest
         {
           return null;
         }
-      }, false, queryContext
+      },
+      queryContext
   );
 
   final Query unionQuery = new FakeQuery(
@@ -141,7 +144,8 @@ public class LoggingRequestLoggerTest
         {
           return null;
         }
-      }, false, queryContext
+      },
+      queryContext
   );
 
 
@@ -325,11 +329,10 @@ class FakeQuery extends BaseQuery<Object>
   public FakeQuery(
       DataSource dataSource,
       QuerySegmentSpec querySegmentSpec,
-      boolean descending,
       Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec, descending, context);
+    super(dataSource, querySegmentSpec, context);
   }
 
   @Override
diff --git a/services/src/main/java/org/apache/druid/cli/DumpSegment.java b/services/src/main/java/org/apache/druid/cli/DumpSegment.java
index 3c663e22488..3f8cfcb94a4 100644
--- a/services/src/main/java/org/apache/druid/cli/DumpSegment.java
+++ b/services/src/main/java/org/apache/druid/cli/DumpSegment.java
@@ -48,7 +48,6 @@ import org.apache.druid.guice.annotations.Json;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.jackson.JacksonUtils;
@@ -71,13 +70,14 @@ import org.apache.druid.segment.BaseObjectColumnValueSelector;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexSegment;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.SimpleAscendingOffset;
-import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.BaseColumn;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnIndexSupplier;
@@ -97,7 +97,6 @@ import org.apache.druid.server.ResourceIdPopulatingQueryRunner;
 import org.apache.druid.timeline.SegmentId;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
-import org.joda.time.chrono.ISOChronology;
 import org.roaringbitmap.IntIterator;
 
 import java.io.File;
@@ -197,7 +196,7 @@ public class DumpSegment extends GuiceRunnable
     try (final QueryableIndex index = indexIO.loadIndex(new File(directory))) {
       switch (dumpType) {
         case ROWS:
-          runDump(injector, index);
+          runDump(injector, outputFileName, index, getColumnsToInclude(index), filterJson, timeISO8601);
           break;
         case METADATA:
           runMetadata(injector, index);
@@ -226,6 +225,16 @@ public class DumpSegment extends GuiceRunnable
     }
   }
 
+  private List<String> getColumnsToInclude(final QueryableIndex index)
+  {
+    return getColumnsToInclude(index, columnNamesFromCli);
+  }
+
+  private <T> T withOutputStream(Function<OutputStream, T> f) throws IOException
+  {
+    return withOutputStream(f, outputFileName);
+  }
+
   private void runMetadata(final Injector injector, final QueryableIndex index) throws IOException
   {
     final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class))
@@ -274,77 +283,74 @@ public class DumpSegment extends GuiceRunnable
     );
   }
 
-  private void runDump(final Injector injector, final QueryableIndex index) throws IOException
+  @VisibleForTesting
+  public static void runDump(
+      final Injector injector,
+      final String outputFileName,
+      final QueryableIndex index,
+      final List<String> columnNames,
+      final String filterJson,
+      final boolean timeISO8601
+  )
+      throws IOException
   {
     final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
     final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index);
-    final List<String> columnNames = getColumnsToInclude(index);
     final DimFilter filter = filterJson != null ? objectMapper.readValue(filterJson, DimFilter.class) : null;
 
-    final Sequence<Cursor> cursors = adapter.makeCursors(
-        Filters.toFilter(filter),
-        index.getDataInterval().withChronology(ISOChronology.getInstanceUTC()),
-        VirtualColumns.EMPTY,
-        Granularities.ALL,
-        false,
-        null
-    );
+    final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+                                                     .setFilter(Filters.toFilter(filter))
+                                                     .build();
 
-    withOutputStream(
-        new Function<OutputStream, Object>()
-        {
-          @Override
-          public Object apply(final OutputStream out)
+    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
+      final Cursor cursor = cursorHolder.asCursor();
+      if (cursor == null) {
+        return;
+      }
+
+      withOutputStream(
+          new Function<OutputStream, Object>()
           {
-            final Sequence<Object> sequence = Sequences.map(
-                cursors,
-                new Function<Cursor, Object>()
-                {
-                  @Override
-                  public Object apply(Cursor cursor)
-                  {
-                    ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
-                    final List<BaseObjectColumnValueSelector> selectors = columnNames
-                        .stream()
-                        .map(columnSelectorFactory::makeColumnValueSelector)
-                        .collect(Collectors.toList());
+            @Override
+            public Object apply(final OutputStream out)
+            {
+              ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory();
+              final List<BaseObjectColumnValueSelector> selectors = columnNames
+                  .stream()
+                  .map(columnSelectorFactory::makeColumnValueSelector)
+                  .collect(Collectors.toList());
 
-                    while (!cursor.isDone()) {
-                      final Map<String, Object> row = Maps.newLinkedHashMap();
+              while (!cursor.isDone()) {
+                final Map<String, Object> row = Maps.newLinkedHashMap();
 
-                      for (int i = 0; i < columnNames.size(); i++) {
-                        final String columnName = columnNames.get(i);
-                        final Object value = selectors.get(i).getObject();
+                for (int i = 0; i < columnNames.size(); i++) {
+                  final String columnName = columnNames.get(i);
+                  final Object value = selectors.get(i).getObject();
 
-                        if (timeISO8601 && columnNames.get(i).equals(ColumnHolder.TIME_COLUMN_NAME)) {
-                          row.put(columnName, new DateTime(value, DateTimeZone.UTC).toString());
-                        } else {
-                          row.put(columnName, value);
-                        }
-                      }
-
-                      try {
-                        out.write(objectMapper.writeValueAsBytes(row));
-                        out.write('\n');
-                      }
-                      catch (IOException e) {
-                        throw new RuntimeException(e);
-                      }
-
-                      cursor.advance();
-                    }
-
-                    return null;
+                  if (timeISO8601 && columnNames.get(i).equals(ColumnHolder.TIME_COLUMN_NAME)) {
+                    row.put(columnName, new DateTime(value, DateTimeZone.UTC).toString());
+                  } else {
+                    row.put(columnName, value);
                   }
                 }
-            );
 
-            evaluateSequenceForSideEffects(sequence);
+                try {
+                  out.write(objectMapper.writeValueAsBytes(row));
+                  out.write('\n');
+                }
+                catch (IOException e) {
+                  throw new RuntimeException(e);
+                }
 
-            return null;
-          }
-        }
-    );
+                cursor.advance();
+              }
+
+              return null;
+            }
+          },
+          outputFileName
+      );
+    }
   }
 
   @VisibleForTesting
@@ -687,9 +693,10 @@ public class DumpSegment extends GuiceRunnable
     );
   }
 
-  private List<String> getColumnsToInclude(final QueryableIndex index)
+  @VisibleForTesting
+  public static List<String> getColumnsToInclude(final QueryableIndex index, List<String> columns)
   {
-    final Set<String> columnNames = Sets.newLinkedHashSet(columnNamesFromCli);
+    final Set<String> columnNames = Sets.newLinkedHashSet(columns);
 
     // Empty columnNames => include all columns.
     if (columnNames.isEmpty()) {
@@ -707,11 +714,6 @@ public class DumpSegment extends GuiceRunnable
     return ImmutableList.copyOf(columnNames);
   }
 
-  private <T> T withOutputStream(Function<OutputStream, T> f) throws IOException
-  {
-    return withOutputStream(f, outputFileName);
-  }
-
   @SuppressForbidden(reason = "System#out")
   private static <T> T withOutputStream(Function<OutputStream, T> f, String outputFileName) throws IOException
   {
diff --git a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java
index 75c8bb6b143..0ca08523fb2 100644
--- a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java
+++ b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java
@@ -115,6 +115,44 @@ public class DumpSegmentTest extends InitializedNullHandlingTest
     Assert.assertSame(expected, actual);
   }
 
+  @Test
+  public void testDumpRows() throws Exception
+  {
+    Injector injector = Mockito.mock(Injector.class);
+    ObjectMapper mapper = TestHelper.makeJsonMapper();
+    mapper.registerModules(BuiltInTypesModule.getJacksonModulesList());
+    mapper.setInjectableValues(
+        new InjectableValues.Std()
+            .addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE)
+            .addValue(ObjectMapper.class.getName(), mapper)
+            .addValue(DefaultColumnFormatConfig.class, new DefaultColumnFormatConfig(null, null))
+    );
+    Mockito.when(injector.getInstance(Key.get(ObjectMapper.class, Json.class))).thenReturn(mapper);
+    Mockito.when(injector.getInstance(DefaultColumnFormatConfig.class)).thenReturn(new DefaultColumnFormatConfig(null, null));
+
+    List<Segment> segments = createSegments(tempFolder, closer);
+    QueryableIndex queryableIndex = segments.get(0).asQueryableIndex();
+
+    File outputFile = tempFolder.newFile();
+
+    DumpSegment.runDump(
+        injector,
+        outputFile.getPath(),
+        queryableIndex,
+        DumpSegment.getColumnsToInclude(queryableIndex, Collections.emptyList()),
+        null,
+        false
+    );
+    final byte[] fileBytes = Files.readAllBytes(outputFile.toPath());
+    final String output = StringUtils.fromUtf8(fileBytes);
+    final String expected = "{\"__time\":1609459200000,\"count\":1,\"nest\":{\"x\":200,\"y\":2.2}}\n"
+                            + "{\"__time\":1609459200000,\"count\":1,\"nest\":{\"x\":400,\"y\":1.1,\"z\":\"a\"}}\n"
+                            + "{\"__time\":1609459200000,\"count\":1,\"nest\":{\"x\":200,\"z\":\"b\"}}\n"
+                            + "{\"__time\":1609459200000,\"count\":1,\"nest\":{\"x\":100,\"y\":1.1,\"z\":\"a\"}}\n"
+                            + "{\"__time\":1609459200000,\"count\":1,\"nest\":{\"y\":3.3,\"z\":\"b\"}}\n";
+    Assert.assertEquals(expected, output);
+  }
+
   @Test
   public void testDumpBitmap() throws IOException
   {
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
index bec5335f7c9..dc127fa7f86 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
@@ -51,6 +51,8 @@ import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.query.DataSource;
 import org.apache.druid.query.FilteredDataSource;
 import org.apache.druid.query.JoinDataSource;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.TableDataSource;
@@ -1614,7 +1616,7 @@ public class DruidQuery
     final DataSource newDataSource = dataSourceFiltrationPair.lhs;
     final Filtration filtration = dataSourceFiltrationPair.rhs;
 
-    final List<ScanQuery.OrderBy> orderByColumns;
+    final List<OrderBy> orderByColumns;
     long scanOffset = 0L;
     long scanLimit = 0L;
 
@@ -1634,11 +1636,11 @@ public class DruidQuery
 
       orderByColumns = sorting.getOrderBys().stream().map(
           orderBy ->
-              new ScanQuery.OrderBy(
+              new OrderBy(
                   orderBy.getDimension(),
                   orderBy.getDirection() == OrderByColumnSpec.Direction.DESCENDING
-                  ? ScanQuery.Order.DESCENDING
-                  : ScanQuery.Order.ASCENDING
+                  ? Order.DESCENDING
+                  : Order.ASCENDING
               )
       ).collect(Collectors.toList());
     } else {
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
index 514695c4afe..813722e6990 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
@@ -36,13 +36,13 @@ import org.apache.druid.error.DruidException;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.TableDataSource;
 import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory;
 import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.groupby.GroupByQuery;
-import org.apache.druid.query.scan.ScanQuery;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.sql.calcite.CalciteCatalogIngestionDmlTest.CatalogIngestionDmlComponentSupplier;
@@ -581,8 +581,8 @@ public abstract class CalciteCatalogIngestionDmlTest extends CalciteIngestionDml
                 )
                 .orderBy(
                     ImmutableList.of(
-                        new ScanQuery.OrderBy("b", ScanQuery.Order.ASCENDING),
-                        new ScanQuery.OrderBy("d", ScanQuery.Order.ASCENDING)
+                        OrderBy.ascending("b"),
+                        OrderBy.ascending("d")
                     )
                 )
                 // Scan query lists columns in alphabetical order independent of the
@@ -643,7 +643,7 @@ public abstract class CalciteCatalogIngestionDmlTest extends CalciteIngestionDml
                 )
                 .orderBy(
                     ImmutableList.of(
-                        new ScanQuery.OrderBy("b", ScanQuery.Order.ASCENDING)
+                        OrderBy.ascending("b")
                     )
                 )
                 // Scan query lists columns in alphabetical order independent of the
@@ -708,7 +708,7 @@ public abstract class CalciteCatalogIngestionDmlTest extends CalciteIngestionDml
                 )
                 .orderBy(
                     ImmutableList.of(
-                        new ScanQuery.OrderBy("e", ScanQuery.Order.ASCENDING)
+                        OrderBy.ascending("e")
                     )
                 )
                 // Scan query lists columns in alphabetical order independent of the
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java
index ba7b19311ce..ed4aa6d91eb 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java
@@ -33,6 +33,7 @@ import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.granularity.GranularityType;
 import org.apache.druid.java.util.common.jackson.JacksonUtils;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@@ -727,9 +728,9 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
         )
         .orderBy(
             ImmutableList.of(
-                new ScanQuery.OrderBy("v0", ScanQuery.Order.ASCENDING),
-                new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING),
-                new ScanQuery.OrderBy("v1", ScanQuery.Order.ASCENDING)
+                OrderBy.ascending("v0"),
+                OrderBy.ascending("dim1"),
+                OrderBy.ascending("v1")
             )
         )
         .context(
@@ -930,9 +931,9 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
                 )
                 .orderBy(
                     ImmutableList.of(
-                        new ScanQuery.OrderBy("v0", ScanQuery.Order.ASCENDING),
-                        new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING),
-                        new ScanQuery.OrderBy("v1", ScanQuery.Order.ASCENDING)
+                        OrderBy.ascending("v0"),
+                        OrderBy.ascending("dim1"),
+                        OrderBy.ascending("v1")
                     )
                 )
                 .context(queryContextWithGranularity(Granularities.DAY))
@@ -971,9 +972,9 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
                 )
                 .orderBy(
                     ImmutableList.of(
-                        new ScanQuery.OrderBy("v0", ScanQuery.Order.ASCENDING),
-                        new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING),
-                        new ScanQuery.OrderBy("v1", ScanQuery.Order.ASCENDING)
+                        OrderBy.ascending("v0"),
+                        OrderBy.ascending("dim1"),
+                        OrderBy.ascending("v1")
                     )
                 )
                 .context(queryContextWithGranularity(Granularities.DAY))
@@ -1021,8 +1022,8 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
                 .virtualColumns(expressionVirtualColumn("v0", "floor(\"m1\")", ColumnType.FLOAT))
                 .orderBy(
                     ImmutableList.of(
-                        new ScanQuery.OrderBy("v0", ScanQuery.Order.ASCENDING),
-                        new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING)
+                        OrderBy.ascending("v0"),
+                        OrderBy.ascending("dim1")
                     )
                 )
                 .context(queryContextWithGranularity(Granularities.DAY))
@@ -1210,9 +1211,9 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
         )
         .orderBy(
             ImmutableList.of(
-                new ScanQuery.OrderBy("v0", ScanQuery.Order.ASCENDING),
-                new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING),
-                new ScanQuery.OrderBy("v1", ScanQuery.Order.ASCENDING)
+                OrderBy.ascending("v0"),
+                OrderBy.ascending("dim1"),
+                OrderBy.ascending("v1")
             )
         )
         .context(
@@ -1325,8 +1326,8 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
                 .columns("v0", "z")
                 .orderBy(
                     ImmutableList.of(
-                        new ScanQuery.OrderBy("v0", ScanQuery.Order.ASCENDING),
-                        new ScanQuery.OrderBy("z", ScanQuery.Order.ASCENDING)
+                        OrderBy.ascending("v0"),
+                        OrderBy.ascending("z")
                     )
                 )
                 .context(PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT)
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
index 599f09d3a03..546776575f2 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
@@ -41,6 +41,7 @@ import org.apache.druid.query.GlobalTableDataSource;
 import org.apache.druid.query.InlineDataSource;
 import org.apache.druid.query.JoinDataSource;
 import org.apache.druid.query.LookupDataSource;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryContext;
 import org.apache.druid.query.QueryContexts;
@@ -5047,7 +5048,7 @@ public class CalciteJoinQueryTest extends BaseCalciteQueryTest
     //
     // This test's goal is to ensure that the join filter rewrites function correctly when there are
     // unoptimized filters in the join query. The rewrite logic must apply to the optimized form of the filters,
-    // as this is what will be passed to HashJoinSegmentAdapter.makeCursors(), where the result of the join
+    // as this is what will be passed to HashJoinSegmentAdapter.makeCursor(), where the result of the join
     // filter pre-analysis is used.
     //
     // A native query is used because the filter types where we support optimization are the AND/OR/NOT and
@@ -5879,7 +5880,7 @@ public class CalciteJoinQueryTest extends BaseCalciteQueryTest
                     )
                 )
                 .columns("__time")
-                .order(ScanQuery.Order.ASCENDING)
+                .order(Order.ASCENDING)
                 .context(context)
                 .build()
         ),
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 84b437c20e0..7566d7f7786 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -44,6 +44,7 @@ import org.apache.druid.query.InlineDataSource;
 import org.apache.druid.query.JoinDataSource;
 import org.apache.druid.query.LookupDataSource;
 import org.apache.druid.query.OperatorFactoryBuilders;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.QueryDataSource;
@@ -15282,7 +15283,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
                     "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
                 .limit(1)
                 .columns(ImmutableList.of("__time", "m1"))
-                .order(ScanQuery.Order.ASCENDING)
+                .order(Order.ASCENDING)
                 .build()
         ),
         ImmutableList.of(
@@ -15324,7 +15325,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
                     "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
                 .limit(1)
                 .columns(ImmutableList.of("__time", "m1"))
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .build()
         ),
         ImmutableList.of(
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java
index 721eec21900..cb819c3634a 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java
@@ -30,6 +30,7 @@ import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.granularity.GranularityType;
 import org.apache.druid.java.util.common.jackson.JacksonUtils;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
@@ -585,8 +586,8 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
                 .virtualColumns(expressionVirtualColumn("v0", "floor(\"m1\")", ColumnType.FLOAT))
                 .orderBy(
                     ImmutableList.of(
-                        new ScanQuery.OrderBy("v0", ScanQuery.Order.ASCENDING),
-                        new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING)
+                        OrderBy.ascending("v0"),
+                        OrderBy.ascending("dim1")
                     )
                 )
                 .context(
@@ -757,11 +758,7 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
         .dataSource("foo")
         .intervals(querySegmentSpec(Filtration.eternity()))
         .columns("__time", "cnt", "dim1", "dim2", "dim3", "m1", "m2", "unique_dim1")
-        .orderBy(
-            ImmutableList.of(
-                new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING)
-            )
-        )
+        .orderBy(ImmutableList.of(OrderBy.ascending("dim1")))
         .columnTypes(LONG, LONG, STRING, STRING, STRING, FLOAT, DOUBLE, ofComplex("hyperUnique"))
         .context(
             queryJsonMapper.readValue(
@@ -842,10 +839,10 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
         .limit(10)
         .orderBy(
             ImmutableList.of(
-                new ScanQuery.OrderBy("__time", ScanQuery.Order.ASCENDING),
-                new ScanQuery.OrderBy("dim1", ScanQuery.Order.ASCENDING),
-                new ScanQuery.OrderBy("dim3", ScanQuery.Order.ASCENDING),
-                new ScanQuery.OrderBy("dim2", ScanQuery.Order.ASCENDING)
+                OrderBy.ascending("__time"),
+                OrderBy.ascending("dim1"),
+                OrderBy.ascending("dim3"),
+                OrderBy.ascending("dim2")
             )
         )
         .columnTypes(LONG, LONG, STRING, STRING, STRING, FLOAT, DOUBLE, ColumnType.ofComplex("hyperUnique"))
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
index 6a0f1742155..6e06c5d4b5d 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
@@ -32,6 +32,7 @@ import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.InlineDataSource;
 import org.apache.druid.query.LookupDataSource;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
@@ -1443,7 +1444,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .intervals(querySegmentSpec(Filtration.eternity()))
                 .columns(ImmutableList.of("__time", "cnt", "dim1", "dim2", "dim3", "m1", "m2", "unique_dim1"))
                 .limit(2)
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(QUERY_CONTEXT_DEFAULT)
                 .build()
@@ -1469,7 +1470,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .intervals(querySegmentSpec(Filtration.eternity()))
                 .columns(ImmutableList.of("__time", "cnt", "dim1", "dim2", "dim3", "m1", "m2", "unique_dim1"))
                 .limit(Long.MAX_VALUE)
-                .order(ScanQuery.Order.ASCENDING)
+                .order(Order.ASCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(QUERY_CONTEXT_DEFAULT)
                 .build()
@@ -1519,7 +1520,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .intervals(querySegmentSpec(Filtration.eternity()))
                 .columns(ImmutableList.of("__time", "dim1"))
                 .limit(2)
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(QUERY_CONTEXT_DEFAULT)
                 .build()
@@ -1545,7 +1546,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .intervals(querySegmentSpec(Filtration.eternity()))
                 .columns(ImmutableList.of("dim1"))
                 .limit(2)
-                .order(ScanQuery.Order.NONE)
+                .order(Order.NONE)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(QUERY_CONTEXT_DEFAULT)
                 .build()
@@ -1569,7 +1570,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .intervals(querySegmentSpec(Filtration.eternity()))
                 .columns(ImmutableList.of("__time", "dim1"))
                 .limit(2)
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(OUTER_LIMIT_CONTEXT)
                 .build()
@@ -1594,7 +1595,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .columns(ImmutableList.of("__time", "dim1"))
                 .offset(1)
                 .limit(2)
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(OUTER_LIMIT_CONTEXT)
                 .build()
@@ -1619,7 +1620,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .columns(ImmutableList.of("__time", "dim1"))
                 .offset(1)
                 .limit(1)
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(OUTER_LIMIT_CONTEXT)
                 .build()
@@ -1643,7 +1644,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .columns(ImmutableList.of("__time", "dim1"))
                 .offset(1)
                 .limit(2)
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(OUTER_LIMIT_CONTEXT)
                 .build()
@@ -1815,7 +1816,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .virtualColumns(expressionVirtualColumn("v0", "concat('beep ',\"dim1\")", ColumnType.STRING))
                 .columns(ImmutableList.of("__time", "v0"))
                 .limit(2)
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(QUERY_CONTEXT_DEFAULT)
                 .build()
@@ -1843,7 +1844,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .intervals(querySegmentSpec(Filtration.eternity()))
                 .virtualColumns(expressionVirtualColumn("v0", "concat('beep ',\"dim1\")", ColumnType.STRING))
                 .columns(ImmutableList.of("v0"))
-                .order(ScanQuery.Order.NONE)
+                .order(Order.NONE)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(QUERY_CONTEXT_DEFAULT)
                 .build()
@@ -1871,7 +1872,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                 .virtualColumns(expressionVirtualColumn("v0", "concat('beep ',\"dim1\")", ColumnType.STRING))
                 .columns(ImmutableList.of("__time", "v0"))
                 .limit(2)
-                .order(ScanQuery.Order.DESCENDING)
+                .order(Order.DESCENDING)
                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                 .context(QUERY_CONTEXT_DEFAULT)
                 .build()
@@ -1899,7 +1900,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
                             .intervals(querySegmentSpec(Filtration.eternity()))
                             .columns(ImmutableList.of("__time", "dim1"))
                             .limit(4)
-                            .order(ScanQuery.Order.DESCENDING)
+                            .order(Order.DESCENDING)
                             .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                             .context(QUERY_CONTEXT_DEFAULT)
                             .build()
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
index 1452358950a..03d9357c7d4 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
@@ -37,6 +37,7 @@ import org.apache.druid.java.util.common.granularity.PeriodGranularity;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.JoinDataSource;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.QueryRunnerFactoryConglomerate;
@@ -1310,7 +1311,7 @@ public class CalciteSubqueryTest extends BaseCalciteQueryTest
                                 .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                                 .offset(6L)
                                 .limit(1L)
-                                .order(ScanQuery.Order.DESCENDING)
+                                .order(Order.DESCENDING)
                                 .columns("__time", "channel")
                                 .context(QUERY_CONTEXT_DEFAULT)
                                 .build()

From c968e731715cd60f563563c06de2ccd068ca0178 Mon Sep 17 00:00:00 2001
From: Edgar Melendrez <evmelendrez@gmail.com>
Date: Fri, 16 Aug 2024 11:39:57 -0700
Subject: [PATCH 22/99] [Docs] updating transformation during ingestion
 tutorial (#16845)

* first major revision of tutorial

* more edits

* re-ID the file to reflect new content + redirect

* renaming file

* Apply suggestions from code review

Co-authored-by: Victoria Lim <vtlim@users.noreply.github.com>

* addressing suggestions

* adding column names

* Update docs/tutorials/tutorial-transform.md

* Update docs/tutorials/tutorial-transform.md

* Addressing suggestions

* Apply suggestions from code review

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* adding trademark logo and moving paragraph

* decided to shorten final paragraph

---------

Co-authored-by: Victoria Lim <vtlim@users.noreply.github.com>
Co-authored-by: Benedict Jin <asdf2014@apache.org>
Co-authored-by: Katya Macedo <38017980+ektravel@users.noreply.github.com>
---
 docs/tutorials/tutorial-transform-spec.md | 156 ----------------------
 docs/tutorials/tutorial-transform.md      | 103 ++++++++++++++
 website/redirects.js                      |   4 +
 website/sidebars.json                     |   2 +-
 4 files changed, 108 insertions(+), 157 deletions(-)
 delete mode 100644 docs/tutorials/tutorial-transform-spec.md
 create mode 100644 docs/tutorials/tutorial-transform.md

diff --git a/docs/tutorials/tutorial-transform-spec.md b/docs/tutorials/tutorial-transform-spec.md
deleted file mode 100644
index 4ce320e3c08..00000000000
--- a/docs/tutorials/tutorial-transform-spec.md
+++ /dev/null
@@ -1,156 +0,0 @@
----
-id: tutorial-transform-spec
-title: Transform input data
-sidebar_label: Transform input data
----
-
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~   http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing,
-  ~ software distributed under the License is distributed on an
-  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  ~ KIND, either express or implied.  See the License for the
-  ~ specific language governing permissions and limitations
-  ~ under the License.
-  -->
-
-
-This tutorial will demonstrate how to use transform specs to filter and transform input data during ingestion.
-
-For this tutorial, we'll assume you've already downloaded Apache Druid as described in
-the [single-machine quickstart](index.md) and have it running on your local machine.
-
-It will also be helpful to have finished [Load a file](../tutorials/tutorial-batch.md) and [Query data](../tutorials/tutorial-query.md) tutorials.
-
-## Sample data
-
-We've included sample data for this tutorial at `quickstart/tutorial/transform-data.json`, reproduced here for convenience:
-
-```json
-{"timestamp":"2018-01-01T07:01:35Z","animal":"octopus",  "location":1, "number":100}
-{"timestamp":"2018-01-01T05:01:35Z","animal":"mongoose", "location":2,"number":200}
-{"timestamp":"2018-01-01T06:01:35Z","animal":"snake", "location":3, "number":300}
-{"timestamp":"2018-01-01T01:01:35Z","animal":"lion", "location":4, "number":300}
-```
-
-## Load data with transform specs
-
-We will ingest the sample data using the following spec, which demonstrates the use of transform specs:
-
-```json
-{
-  "type" : "index_parallel",
-  "spec" : {
-    "dataSchema" : {
-      "dataSource" : "transform-tutorial",
-      "timestampSpec": {
-        "column": "timestamp",
-        "format": "iso"
-      },
-      "dimensionsSpec" : {
-        "dimensions" : [
-          "animal",
-          { "name": "location", "type": "long" }
-        ]
-      },
-      "metricsSpec" : [
-        { "type" : "count", "name" : "count" },
-        { "type" : "longSum", "name" : "number", "fieldName" : "number" },
-        { "type" : "longSum", "name" : "triple-number", "fieldName" : "triple-number" }
-      ],
-      "granularitySpec" : {
-        "type" : "uniform",
-        "segmentGranularity" : "week",
-        "queryGranularity" : "minute",
-        "intervals" : ["2018-01-01/2018-01-03"],
-        "rollup" : true
-      },
-      "transformSpec": {
-        "transforms": [
-          {
-            "type": "expression",
-            "name": "animal",
-            "expression": "concat('super-', animal)"
-          },
-          {
-            "type": "expression",
-            "name": "triple-number",
-            "expression": "number * 3"
-          }
-        ],
-        "filter": {
-          "type":"or",
-          "fields": [
-            { "type": "selector", "dimension": "animal", "value": "super-mongoose" },
-            { "type": "selector", "dimension": "triple-number", "value": "300" },
-            { "type": "selector", "dimension": "location", "value": "3" }
-          ]
-        }
-      }
-    },
-    "ioConfig" : {
-      "type" : "index_parallel",
-      "inputSource" : {
-        "type" : "local",
-        "baseDir" : "quickstart/tutorial",
-        "filter" : "transform-data.json"
-      },
-      "inputFormat" : {
-        "type" :"json"
-      },
-      "appendToExisting" : false
-    },
-    "tuningConfig" : {
-      "type" : "index_parallel",
-      "partitionsSpec": {
-        "type": "dynamic"
-      },
-      "maxRowsInMemory" : 25000
-    }
-  }
-}
-```
-
-In the transform spec, we have two expression transforms:
-* `super-animal`: prepends "super-" to the values in the `animal` column. This will override the `animal` column with the transformed version, since the transform's name is `animal`.
-* `triple-number`: multiplies the `number` column by 3. This will create a new `triple-number` column. Note that we are ingesting both the original and the transformed column.
-
-Additionally, we have an OR filter with three clauses:
-* `super-animal` values that match "super-mongoose"
-* `triple-number` values that match 300
-* `location` values that match 3
-
-This filter selects the first 3 rows, and it will exclude the final "lion" row in the input data. Note that the filter is applied after the transformation.
-
-Let's submit this task now, which has been included at `quickstart/tutorial/transform-index.json`:
-
-```bash
-bin/post-index-task --file quickstart/tutorial/transform-index.json --url http://localhost:8081
-```
-
-## Query the transformed data
-
-Let's run `bin/dsql` and issue a `select * from "transform-tutorial";` query to see what was ingested:
-
-```bash
-dsql> select * from "transform-tutorial";
-┌──────────────────────────┬────────────────┬───────┬──────────┬────────┬───────────────┐
-│ __time                   │ animal         │ count │ location │ number │ triple-number │
-├──────────────────────────┼────────────────┼───────┼──────────┼────────┼───────────────┤
-│ 2018-01-01T05:01:00.000Z │ super-mongoose │     1 │        2 │    200 │           600 │
-│ 2018-01-01T06:01:00.000Z │ super-snake    │     1 │        3 │    300 │           900 │
-│ 2018-01-01T07:01:00.000Z │ super-octopus  │     1 │        1 │    100 │           300 │
-└──────────────────────────┴────────────────┴───────┴──────────┴────────┴───────────────┘
-Retrieved 3 rows in 0.03s.
-```
-
-The "lion" row has been discarded, the `animal` column has been transformed, and we have both the original and transformed `number` column.
diff --git a/docs/tutorials/tutorial-transform.md b/docs/tutorials/tutorial-transform.md
new file mode 100644
index 00000000000..3c9815b82bb
--- /dev/null
+++ b/docs/tutorials/tutorial-transform.md
@@ -0,0 +1,103 @@
+---
+id: tutorial-transform
+title: Transform input data
+sidebar_label: Transform input data
+---
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+
+This tutorial demonstrates how to transform input data during ingestion.
+
+## Prerequisite
+
+Before proceeding, download Apache Druid&circledR; as described in [Quickstart (local)](index.md) and have it running on your local machine. You don't need to load any data into the Druid cluster.
+
+You should be familiar with data querying in Druid. If you haven't already, go through the [Query data](../tutorials/tutorial-query.md) tutorial first.
+
+## Sample data
+
+For this tutorial, you use the following sample data:
+
+```json
+{"timestamp":"2018-01-01T07:01:35Z", "animal":"octopus", "location":1, "number":100}
+{"timestamp":"2018-01-01T05:01:35Z", "animal":"mongoose", "location":2,"number":200}
+{"timestamp":"2018-01-01T06:01:35Z", "animal":"snake", "location":3, "number":300}
+{"timestamp":"2018-01-01T01:01:35Z", "animal":"lion", "location":4, "number":300}
+```
+
+## Transform data during ingestion
+
+Load the sample dataset using the [`INSERT INTO`](../multi-stage-query/reference.md/#insert) statement and the [`EXTERN`](../multi-stage-query/reference.md/#extern-function) function to ingest the data inline. In the [Druid web console](../operations/web-console.md), go to the **Query** view and run the following query:
+
+```sql
+INSERT INTO "transform_tutorial"
+WITH "ext" AS (
+  SELECT *
+  FROM TABLE(EXTERN('{"type":"inline","data":"{\"timestamp\":\"2018-01-01T07:01:35Z\",\"animal\":\"octopus\",  \"location\":1, \"number\":100}\n{\"timestamp\":\"2018-01-01T05:01:35Z\",\"animal\":\"mongoose\", \"location\":2,\"number\":200}\n{\"timestamp\":\"2018-01-01T06:01:35Z\",\"animal\":\"snake\", \"location\":3, \"number\":300}\n{\"timestamp\":\"2018-01-01T01:01:35Z\",\"animal\":\"lion\", \"location\":4, \"number\":300}"}', '{"type":"json"}')) EXTEND ("timestamp" VARCHAR, "animal" VARCHAR, "location" BIGINT, "number" BIGINT)
+)
+SELECT
+  TIME_PARSE("timestamp") AS "__time",
+  TEXTCAT('super-', "animal") AS "animal",
+  "location",
+  "number",
+  "number" * 3 AS "triple-number"
+FROM "ext"
+WHERE (TEXTCAT('super-', "animal") = 'super-mongoose' OR "location" = 3 OR "number" = 100)
+PARTITIONED BY DAY
+```
+
+In the `SELECT` clause, you specify the following transformations:
+* `animal`: prepends "super-" to the values in the `animal` column using the [`TEXTCAT`](../querying/sql-functions.md/#textcat) function. Note that it only ingests the transformed data.
+* `triple-number`: multiplies the `number` column by three and stores the results in a column named `triple-number`. Note that the query ingests both the original and the transformed data.
+
+Additionally, the `WHERE` clause applies the following three OR operators so that the query only ingests the rows where at least one of the following conditions is `true`:
+
+* `TEXTCAT('super-', "animal")` matches "super-mongoose"
+* `location` matches 3
+* `number` matches 100
+
+Once a row passes the filter, the ingestion job applies the transformations. In this example, the filter selects the first three rows because each row meets at least one of the required OR conditions. For the selected rows, the ingestion job ingests the transformed `animal` column, the `location` column, and both the original `number` and the transformed `triple-number` column. The "lion" row doesn't meet any of the conditions, so it is not ingested or transformed.
+
+## Query the transformed data
+
+In the web console, open a new tab in the **Query** view. Run the following query to view the ingested data:
+
+```sql
+SELECT * FROM "transform_tutorial"
+```
+
+Returns the following:
+
+| `__time` | `animal` | `location` | `number` | `triple-number` | 
+| -- | -- | -- | -- | -- |
+| `2018-01-01T05:01:35.000Z` | `super-mongoose` | `2` | `200` | `600` |
+| `2018-01-01T06:01:35.000Z` | `super-snake` | `3` | `300` | `900` |
+| `2018-01-01T07:01:35.000Z` | `super-octopus` | `1` |  `100` | `300` |
+
+Notice how the "lion" row is missing, and how the other three rows that were ingested have transformations applied to them.
+
+## Learn more
+
+See the following topics for more information:
+
+* [All functions](../querying/sql-functions.md) for a list of functions that can be used to transform data. 
+* [Transform spec reference](../ingestion/ingestion-spec.md/#transformspec) to learn more about transforms in JSON-based batch ingestion.
+* [WHERE clause](../querying/sql.md#where) to learn how to specify filters in Druid SQL.
\ No newline at end of file
diff --git a/website/redirects.js b/website/redirects.js
index 4eb2a36c33b..1d479839503 100644
--- a/website/redirects.js
+++ b/website/redirects.js
@@ -20,6 +20,10 @@
 
 
 const Redirects=[
+  {
+    "from": "/docs/latest/tutorials/tutorial-transform-spec",
+    "to": "/docs/latest/tutorials/tutorial-transform"
+  },
   {
     "from": "/docs/latest/development/extensions-core/kafka-extraction-namespace/",
     "to": "/docs/latest/querying/kafka-extraction-namespace"
diff --git a/website/sidebars.json b/website/sidebars.json
index b694e3f11bb..e5304006318 100644
--- a/website/sidebars.json
+++ b/website/sidebars.json
@@ -18,7 +18,7 @@
         "tutorials/tutorial-kafka",
         "tutorials/tutorial-rollup",
         "tutorials/tutorial-ingestion-spec",
-        "tutorials/tutorial-transform-spec",
+        "tutorials/tutorial-transform",
         "tutorials/tutorial-msq-convert-spec"
               ]},
       {"type": "category",

From 422183ee70d6e14e32213289b926eca68e857fa6 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Fri, 16 Aug 2024 14:39:16 -0700
Subject: [PATCH 23/99] Web console: expose handoff API (#16586)

* don't start completions on numbers... it makes numbers hard to enter

* add handoff dialog

* fix placeholder

* Update web-console/src/dialogs/supervisor-handoff-dialog/supervisor-handoff-dialog.tsx

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* Update web-console/src/dialogs/supervisor-handoff-dialog/supervisor-handoff-dialog.tsx

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* Update web-console/src/dialogs/supervisor-handoff-dialog/supervisor-handoff-dialog.tsx

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* feedback fixes

* update snapshot

---------

Co-authored-by: Katya Macedo <38017980+ektravel@users.noreply.github.com>
---
 web-console/src/ace-modes/dsql.ts             |  5 +-
 web-console/src/dialogs/index.ts              |  1 +
 .../task-group-handoff-dialog.tsx             | 65 +++++++++++++++++++
 .../supervisors-view.spec.tsx.snap            |  2 +-
 .../supervisors-view/supervisors-view.tsx     | 48 +++++++++++---
 .../flexible-query-input.tsx                  |  4 ++
 6 files changed, 114 insertions(+), 11 deletions(-)
 create mode 100644 web-console/src/dialogs/task-group-handoff-dialog/task-group-handoff-dialog.tsx

diff --git a/web-console/src/ace-modes/dsql.ts b/web-console/src/ace-modes/dsql.ts
index e57d17d51e7..38cc40124e2 100644
--- a/web-console/src/ace-modes/dsql.ts
+++ b/web-console/src/ace-modes/dsql.ts
@@ -186,7 +186,10 @@ ace.define(
       this.$id = 'ace/mode/dsql';
 
       this.lineCommentStart = '--';
-      this.getCompletions = () => completions;
+      this.getCompletions = (_state: any, _session: any, _pos: any, prefix: string) => {
+        if (/^\d+$/.test(prefix)) return; // Don't start completing if the user is typing a number
+        return completions;
+      };
     };
     oop.inherits(Mode, TextMode);
 
diff --git a/web-console/src/dialogs/index.ts b/web-console/src/dialogs/index.ts
index f7e239b53f4..6a3c336831b 100644
--- a/web-console/src/dialogs/index.ts
+++ b/web-console/src/dialogs/index.ts
@@ -37,4 +37,5 @@ export * from './string-input-dialog/string-input-dialog';
 export * from './supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog';
 export * from './supervisor-table-action-dialog/supervisor-table-action-dialog';
 export * from './table-action-dialog/table-action-dialog';
+export * from './task-group-handoff-dialog/task-group-handoff-dialog';
 export * from './task-table-action-dialog/task-table-action-dialog';
diff --git a/web-console/src/dialogs/task-group-handoff-dialog/task-group-handoff-dialog.tsx b/web-console/src/dialogs/task-group-handoff-dialog/task-group-handoff-dialog.tsx
new file mode 100644
index 00000000000..02e4b066af9
--- /dev/null
+++ b/web-console/src/dialogs/task-group-handoff-dialog/task-group-handoff-dialog.tsx
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FormGroup, Intent, Tag } from '@blueprintjs/core';
+import React, { useState } from 'react';
+
+import { ArrayInput } from '../../components';
+import { Api } from '../../singletons';
+import { AsyncActionDialog } from '../async-action-dialog/async-action-dialog';
+
+export interface TaskGroupHandoffDialogProps {
+  supervisorId: string;
+  onSuccess(): void;
+  onClose(): void;
+}
+
+export function TaskGroupHandoffDialog(props: TaskGroupHandoffDialogProps) {
+  const { supervisorId, onSuccess, onClose } = props;
+  const [groupIds, setGroupIds] = useState<string[]>([]);
+
+  return (
+    <AsyncActionDialog
+      action={async () => {
+        const resp = await Api.instance.post(
+          `/druid/indexer/v1/supervisor/${Api.encodePath(supervisorId)}/taskGroups/handoff`,
+          { taskGroupIds: groupIds.map(x => parseInt(x, 10)) },
+        );
+        return resp.data;
+      }}
+      confirmButtonText="Handoff task groups"
+      successText="Task group handoff has been initiated"
+      failText="Could not initiate handoff for the selected task groups"
+      intent={Intent.PRIMARY}
+      onClose={onClose}
+      onSuccess={onSuccess}
+    >
+      <p>
+        Are you sure you want to initiate early handoff for the given task groups belonging to
+        supervisor <Tag minimal>{supervisorId}</Tag>?
+      </p>
+      <FormGroup label="Task group IDs">
+        <ArrayInput
+          values={groupIds}
+          onChange={v => setGroupIds(v || [])}
+          placeholder="0, 1, 2, ..."
+        />
+      </FormGroup>
+    </AsyncActionDialog>
+  );
+}
diff --git a/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap b/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
index 4dc9526c12c..4b531c0bc44 100644
--- a/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
+++ b/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
@@ -397,7 +397,7 @@ exports[`SupervisorsView matches snapshot 1`] = `
     rowsText="rows"
     showPageJump={false}
     showPageSizeOptions={true}
-    showPagination={true}
+    showPagination={false}
     showPaginationBottom={true}
     showPaginationTop={false}
     sortable={true}
diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx
index 60e340034eb..648186d45bc 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.tsx
+++ b/web-console/src/views/supervisors-view/supervisors-view.tsx
@@ -45,6 +45,7 @@ import {
   SupervisorTableActionDialog,
 } from '../../dialogs';
 import { SupervisorResetOffsetsDialog } from '../../dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog';
+import { TaskGroupHandoffDialog } from '../../dialogs/task-group-handoff-dialog/task-group-handoff-dialog';
 import type {
   IngestionSpec,
   QueryWithContext,
@@ -144,6 +145,7 @@ export interface SupervisorsViewState {
 
   resumeSupervisorId?: string;
   suspendSupervisorId?: string;
+  handoffSupervisorId?: string;
   resetOffsetsSupervisorInfo?: { id: string; type: string };
   resetSupervisorId?: string;
   terminateSupervisorId?: string;
@@ -425,15 +427,25 @@ export class SupervisorsView extends React.PureComponent<
         },
       );
     }
+
+    actions.push({
+      icon: supervisorSuspended ? IconNames.PLAY : IconNames.PAUSE,
+      title: supervisorSuspended ? 'Resume' : 'Suspend',
+      onAction: () =>
+        supervisorSuspended
+          ? this.setState({ resumeSupervisorId: id })
+          : this.setState({ suspendSupervisorId: id }),
+    });
+
+    if (!supervisorSuspended) {
+      actions.push({
+        icon: IconNames.AUTOMATIC_UPDATES,
+        title: 'Handoff early',
+        onAction: () => this.setState({ handoffSupervisorId: id }),
+      });
+    }
+
     actions.push(
-      {
-        icon: supervisorSuspended ? IconNames.PLAY : IconNames.PAUSE,
-        title: supervisorSuspended ? 'Resume' : 'Suspend',
-        onAction: () =>
-          supervisorSuspended
-            ? this.setState({ resumeSupervisorId: id })
-            : this.setState({ suspendSupervisorId: id }),
-      },
       {
         icon: IconNames.STEP_BACKWARD,
         title: `Set ${type === 'kinesis' ? 'sequence numbers' : 'offsets'}`,
@@ -519,6 +531,23 @@ export class SupervisorsView extends React.PureComponent<
     );
   }
 
+  renderTaskGroupHandoffAction() {
+    const { handoffSupervisorId } = this.state;
+    if (!handoffSupervisorId) return;
+
+    return (
+      <TaskGroupHandoffDialog
+        supervisorId={handoffSupervisorId}
+        onClose={() => {
+          this.setState({ handoffSupervisorId: undefined });
+        }}
+        onSuccess={() => {
+          this.supervisorQueryManager.rerunLastQuery();
+        }}
+      />
+    );
+  }
+
   renderResetOffsetsSupervisorAction() {
     const { resetOffsetsSupervisorInfo } = this.state;
     if (!resetOffsetsSupervisorInfo) return;
@@ -659,7 +688,7 @@ export class SupervisorsView extends React.PureComponent<
         ofText=""
         defaultPageSize={SMALL_TABLE_PAGE_SIZE}
         pageSizeOptions={SMALL_TABLE_PAGE_SIZE_OPTIONS}
-        showPagination
+        showPagination={supervisors.length > SMALL_TABLE_PAGE_SIZE}
         columns={[
           {
             Header: twoLines('Supervisor ID', <i>(datasource)</i>),
@@ -1061,6 +1090,7 @@ export class SupervisorsView extends React.PureComponent<
         {this.renderSupervisorTable()}
         {this.renderResumeSupervisorAction()}
         {this.renderSuspendSupervisorAction()}
+        {this.renderTaskGroupHandoffAction()}
         {this.renderResetOffsetsSupervisorAction()}
         {this.renderResetSupervisorAction()}
         {this.renderTerminateSupervisorAction()}
diff --git a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx
index f4be7275944..cb4e9e3b192 100644
--- a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx
+++ b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx
@@ -81,6 +81,10 @@ export class FlexibleQueryInput extends React.PureComponent<
     // Local completions
     {
       getCompletions: (_state, session, pos, prefix, callback) => {
+        if (/^\d+$/.test(prefix)) {
+          callback(null, []); // Don't start completing if the user is typing a number
+          return;
+        }
         const charBeforePrefix = session.getLine(pos.row)[pos.column - prefix.length - 1];
         callback(
           null,

From 806649f8af6110c4a60b3014c9014ef4e012008c Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Fri, 16 Aug 2024 22:41:12 -0700
Subject: [PATCH 24/99] SQL: Fix nullable DATE, TIMESTAMP reduction. (#16915)

Reduction of nullable DATE and TIMESTAMP expressions did not perform
a necessary null check, so would in some cases reduce to
1970-01-01 00:00:00 (epoch) rather than NULL.
---
 .../sql/calcite/planner/DruidRexExecutor.java | 56 ++++++++-------
 .../calcite/planner/DruidRexExecutorTest.java | 71 +++++++++++++++++++
 2 files changed, 103 insertions(+), 24 deletions(-)

diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
index 715c5caaadc..1f73834fdb2 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
@@ -90,33 +90,41 @@ public class DruidRexExecutor implements RexExecutor
             literal = rexBuilder.makeLiteral(exprResult.asBoolean(), constExp.getType(), true);
           }
         } else if (sqlTypeName == SqlTypeName.DATE) {
-          // It is possible for an expression to have a non-null String value but it can return null when parsed
-          // as a primitive long/float/double.
-          // ExprEval.isNumericNull checks whether the parsed primitive value is null or not.
-          if (!constExp.getType().isNullable() && exprResult.isNumericNull()) {
-            throw InvalidSqlInput.exception("Illegal DATE constant [%s]", constExp);
+          if (exprResult.isNumericNull()) {
+            if (constExp.getType().isNullable()) {
+              literal = rexBuilder.makeNullLiteral(constExp.getType());
+            } else {
+              // There can be implicit casts of VARCHAR to TIMESTAMP where the VARCHAR is an invalid timestamp, but the
+              // TIMESTAMP type is not nullable. In this case it's best to throw an error, since it likely means the
+              // user's SQL query contains an invalid literal.
+              throw InvalidSqlInput.exception("Illegal DATE constant [%s]", constExp);
+            }
+          } else {
+            literal = rexBuilder.makeDateLiteral(
+                Calcites.jodaToCalciteDateString(
+                    DateTimes.utc(exprResult.asLong()),
+                    plannerContext.getTimeZone()
+                )
+            );
           }
-
-          literal = rexBuilder.makeDateLiteral(
-              Calcites.jodaToCalciteDateString(
-                  DateTimes.utc(exprResult.asLong()),
-                  plannerContext.getTimeZone()
-              )
-          );
         } else if (sqlTypeName == SqlTypeName.TIMESTAMP) {
-          // It is possible for an expression to have a non-null String value but it can return null when parsed
-          // as a primitive long/float/double.
-          // ExprEval.isNumericNull checks whether the parsed primitive value is null or not.
-          if (!constExp.getType().isNullable() && exprResult.isNumericNull()) {
-            throw InvalidSqlInput.exception("Illegal TIMESTAMP constant [%s]", constExp);
+          if (exprResult.isNumericNull()) {
+            if (constExp.getType().isNullable()) {
+              literal = rexBuilder.makeNullLiteral(constExp.getType());
+            } else {
+              // There can be implicit casts of VARCHAR to TIMESTAMP where the VARCHAR is an invalid timestamp, but the
+              // TIMESTAMP type is not nullable. In this case it's best to throw an error, since it likely means the
+              // user's SQL query contains an invalid literal.
+              throw InvalidSqlInput.exception("Illegal TIMESTAMP constant [%s]", constExp);
+            }
+          } else {
+            literal = Calcites.jodaToCalciteTimestampLiteral(
+                rexBuilder,
+                DateTimes.utc(exprResult.asLong()),
+                plannerContext.getTimeZone(),
+                constExp.getType().getPrecision()
+            );
           }
-
-          literal = Calcites.jodaToCalciteTimestampLiteral(
-              rexBuilder,
-              DateTimes.utc(exprResult.asLong()),
-              plannerContext.getTimeZone(),
-              constExp.getType().getPrecision()
-          );
         } else if (SqlTypeName.NUMERIC_TYPES.contains(sqlTypeName)) {
           final BigDecimal bigDecimal;
 
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
index 072bdd6b6e8..7888ec8090b 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
@@ -37,6 +37,7 @@ import org.apache.calcite.sql.type.BasicSqlType;
 import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
 import org.apache.calcite.sql.type.SqlTypeFamily;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
@@ -46,6 +47,7 @@ import org.apache.druid.sql.calcite.expression.DruidExpression;
 import org.apache.druid.sql.calcite.expression.Expressions;
 import org.apache.druid.sql.calcite.expression.OperatorConversions;
 import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringOperatorConversions;
+import org.apache.druid.sql.calcite.expression.builtin.TimeParseOperatorConversion;
 import org.apache.druid.sql.calcite.schema.DruidSchema;
 import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
 import org.apache.druid.sql.calcite.schema.NamedDruidSchema;
@@ -57,6 +59,7 @@ import org.apache.druid.sql.calcite.util.CalciteTests;
 import org.apache.druid.sql.hook.DruidHookDispatcher;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.easymock.EasyMock;
+import org.joda.time.DateTimeZone;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -139,6 +142,74 @@ public class DruidRexExecutorTest extends InitializedNullHandlingTest
     Assert.assertEquals(new BigDecimal(30L), ((RexLiteral) reduced.get(0)).getValue());
   }
 
+  @Test
+  public void testCastDateReduced()
+  {
+    // CAST('2010-01-01' AS DATE)
+    RexNode call = rexBuilder.makeCall(
+        rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DATE),
+        SqlStdOperatorTable.CAST,
+        Collections.singletonList(rexBuilder.makeLiteral("2010-01-01"))
+    );
+
+    DruidRexExecutor rexy = new DruidRexExecutor(PLANNER_CONTEXT);
+    List<RexNode> reduced = new ArrayList<>();
+    rexy.reduce(rexBuilder, ImmutableList.of(call), reduced);
+    Assert.assertEquals(1, reduced.size());
+    Assert.assertEquals(SqlKind.LITERAL, reduced.get(0).getKind());
+    Assert.assertEquals(
+        rexBuilder.makeDateLiteral(
+            Calcites.jodaToCalciteDateString(
+                DateTimes.of("2010-01-01"),
+                DateTimeZone.UTC
+            )
+        ),
+        reduced.get(0)
+    );
+  }
+
+  @Test
+  public void testTimeParseReduced()
+  {
+    // TIME_PARSE('2010-01-01T02:03:04Z')
+    RexNode call = rexBuilder.makeCall(
+        new TimeParseOperatorConversion().calciteOperator(),
+        rexBuilder.makeLiteral("2010-01-01T02:03:04Z")
+    );
+
+    DruidRexExecutor rexy = new DruidRexExecutor(PLANNER_CONTEXT);
+    List<RexNode> reduced = new ArrayList<>();
+    rexy.reduce(rexBuilder, ImmutableList.of(call), reduced);
+    Assert.assertEquals(1, reduced.size());
+    Assert.assertEquals(SqlKind.LITERAL, reduced.get(0).getKind());
+    Assert.assertEquals(
+        Calcites.jodaToCalciteTimestampLiteral(
+            rexBuilder,
+            DateTimes.of("2010-01-01T02:03:04Z"),
+            DateTimeZone.UTC,
+            DruidTypeSystem.DEFAULT_TIMESTAMP_PRECISION
+        ),
+        reduced.get(0)
+    );
+  }
+
+  @Test
+  public void testTimeParseUnparseableReduced()
+  {
+    // TIME_PARSE('not a timestamp')
+    RexNode call = rexBuilder.makeCall(
+        new TimeParseOperatorConversion().calciteOperator(),
+        rexBuilder.makeLiteral("not a timestamp")
+    );
+
+    DruidRexExecutor rexy = new DruidRexExecutor(PLANNER_CONTEXT);
+    List<RexNode> reduced = new ArrayList<>();
+    rexy.reduce(rexBuilder, ImmutableList.of(call), reduced);
+    Assert.assertEquals(1, reduced.size());
+    Assert.assertEquals(SqlKind.LITERAL, reduced.get(0).getKind());
+    Assert.assertTrue(RexLiteral.isNullLiteral(reduced.get(0)));
+  }
+
   @Test
   public void testComplexNotReduced()
   {

From 688b4cf16488d1f726719f1ade3ed11234ce8c89 Mon Sep 17 00:00:00 2001
From: Benedict Jin <asdf2014@apache.org>
Date: Sun, 18 Aug 2024 12:32:50 +0800
Subject: [PATCH 25/99] Fix flaky test in ParallelMergeCombiningSequenceTest
 (#16907)

---
 .../util/common/guava/ParallelMergeCombiningSequenceTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/processing/src/test/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequenceTest.java b/processing/src/test/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequenceTest.java
index 5b76afb9022..293943dcca9 100644
--- a/processing/src/test/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequenceTest.java
+++ b/processing/src/test/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequenceTest.java
@@ -605,7 +605,7 @@ public class ParallelMergeCombiningSequenceTest
     }
 
     // sleep until timeout
-    Thread.sleep(1000);
+    Thread.sleep(5000);
     Assert.assertTrue(pool.awaitQuiescence(10, TimeUnit.SECONDS));
     Assert.assertTrue(pool.isQuiescent());
     Assert.assertFalse(pool.hasQueuedSubmissions());

From a56b5c018de985fe462c904212f9411ab4e0c6d1 Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Sun, 18 Aug 2024 10:03:45 +0530
Subject: [PATCH 26/99] Propagate TooManyRowsInAWindowFault error message
 properly to the user (#16906)

* Propagate TooManyRowsInAWindowFault error message properly to the user

* Add TooManyRowsInAWindowFault to MSQFaultSerdeTest
---
 .../main/java/org/apache/druid/msq/guice/MSQIndexingModule.java | 2 ++
 .../org/apache/druid/msq/indexing/error/MSQFaultSerdeTest.java  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQIndexingModule.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQIndexingModule.java
index 01a8743dd86..4e28edc3ac1 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQIndexingModule.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQIndexingModule.java
@@ -60,6 +60,7 @@ import org.apache.druid.msq.indexing.error.TooManyClusteredByColumnsFault;
 import org.apache.druid.msq.indexing.error.TooManyColumnsFault;
 import org.apache.druid.msq.indexing.error.TooManyInputFilesFault;
 import org.apache.druid.msq.indexing.error.TooManyPartitionsFault;
+import org.apache.druid.msq.indexing.error.TooManyRowsInAWindowFault;
 import org.apache.druid.msq.indexing.error.TooManyRowsWithSameKeyFault;
 import org.apache.druid.msq.indexing.error.TooManySegmentsInTimeChunkFault;
 import org.apache.druid.msq.indexing.error.TooManyWarningsFault;
@@ -128,6 +129,7 @@ public class MSQIndexingModule implements DruidModule
       TooManyColumnsFault.class,
       TooManyInputFilesFault.class,
       TooManyPartitionsFault.class,
+      TooManyRowsInAWindowFault.class,
       TooManyRowsWithSameKeyFault.class,
       TooManySegmentsInTimeChunkFault.class,
       TooManyWarningsFault.class,
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/error/MSQFaultSerdeTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/error/MSQFaultSerdeTest.java
index cffc0f78a49..0c16ae08af5 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/error/MSQFaultSerdeTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/error/MSQFaultSerdeTest.java
@@ -85,6 +85,7 @@ public class MSQFaultSerdeTest
     assertFaultSerde(new TooManyClusteredByColumnsFault(10, 8, 1));
     assertFaultSerde(new TooManyInputFilesFault(15, 10, 5));
     assertFaultSerde(new TooManyPartitionsFault(10));
+    assertFaultSerde(new TooManyRowsInAWindowFault(10, 20));
     assertFaultSerde(new TooManyRowsWithSameKeyFault(Arrays.asList("foo", 123), 1, 2));
     assertFaultSerde(new TooManySegmentsInTimeChunkFault(DateTimes.nowUtc(), 10, 1, Granularities.ALL));
     assertFaultSerde(new TooManyWarningsFault(10, "the error"));

From 4e33ce2b21264a3059c9d677585e6bfecf4e5ff3 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Sun, 18 Aug 2024 15:11:28 -0700
Subject: [PATCH 27/99] fix collapsing in column tree (#16910)

---
 licenses.yaml                                 |  2 +-
 web-console/package-lock.json                 | 14 ++++----
 web-console/package.json                      |  2 +-
 .../multi-axis-chart-echarts-module.ts        |  2 +-
 .../modules/pie-chart-echarts-module.ts       |  4 +--
 .../modules/time-chart-echarts-module.ts      |  2 +-
 .../column-tree/column-tree.spec.tsx          |  2 +-
 .../column-tree/column-tree.tsx               | 32 +++----------------
 .../views/workbench-view/workbench-view.tsx   | 11 +++----
 9 files changed, 22 insertions(+), 49 deletions(-)

diff --git a/licenses.yaml b/licenses.yaml
index b6e07572341..304880db3db 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -5085,7 +5085,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Imply Data
-version: 0.22.21
+version: 0.22.22
 
 ---
 
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index 20396ed7265..44d3cbae2a4 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -14,7 +14,7 @@
         "@blueprintjs/datetime2": "^2.3.7",
         "@blueprintjs/icons": "^5.10.0",
         "@blueprintjs/select": "^5.2.1",
-        "@druid-toolkit/query": "^0.22.21",
+        "@druid-toolkit/query": "^0.22.22",
         "@druid-toolkit/visuals-core": "^0.3.3",
         "@druid-toolkit/visuals-react": "^0.3.3",
         "@fontsource/open-sans": "^5.0.28",
@@ -989,9 +989,9 @@
       }
     },
     "node_modules/@druid-toolkit/query": {
-      "version": "0.22.21",
-      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.21.tgz",
-      "integrity": "sha512-4k0NGO2Ay90naSO8nyivPPvvhz73D/OkCo6So3frmPDLFfw5CYKSvAhy4RadtnLMZPwsnlVREjAmqbvBsHqgjQ==",
+      "version": "0.22.22",
+      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.22.tgz",
+      "integrity": "sha512-QCzIFr0H6uk5KsGEKO/Vk1fWWurf54ENagFCygsKVk3p2sCRmpZ7WGKeQ5i3AFh2847tOeopZI0OniEv2yTLGQ==",
       "dependencies": {
         "tslib": "^2.5.2"
       }
@@ -19093,9 +19093,9 @@
       "dev": true
     },
     "@druid-toolkit/query": {
-      "version": "0.22.21",
-      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.21.tgz",
-      "integrity": "sha512-4k0NGO2Ay90naSO8nyivPPvvhz73D/OkCo6So3frmPDLFfw5CYKSvAhy4RadtnLMZPwsnlVREjAmqbvBsHqgjQ==",
+      "version": "0.22.22",
+      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.22.tgz",
+      "integrity": "sha512-QCzIFr0H6uk5KsGEKO/Vk1fWWurf54ENagFCygsKVk3p2sCRmpZ7WGKeQ5i3AFh2847tOeopZI0OniEv2yTLGQ==",
       "requires": {
         "tslib": "^2.5.2"
       }
diff --git a/web-console/package.json b/web-console/package.json
index 6d8c0c6448a..9c9f38d24cb 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -68,7 +68,7 @@
     "@blueprintjs/datetime2": "^2.3.7",
     "@blueprintjs/icons": "^5.10.0",
     "@blueprintjs/select": "^5.2.1",
-    "@druid-toolkit/query": "^0.22.21",
+    "@druid-toolkit/query": "^0.22.22",
     "@druid-toolkit/visuals-core": "^0.3.3",
     "@druid-toolkit/visuals-react": "^0.3.3",
     "@fontsource/open-sans": "^5.0.28",
diff --git a/web-console/src/views/explore-view/modules/multi-axis-chart-echarts-module.ts b/web-console/src/views/explore-view/modules/multi-axis-chart-echarts-module.ts
index c315b4c9444..ee87e4932b8 100644
--- a/web-console/src/views/explore-view/modules/multi-axis-chart-echarts-module.ts
+++ b/web-console/src/views/explore-view/modules/multi-axis-chart-echarts-module.ts
@@ -197,7 +197,7 @@ export default typedVisualModule({
                       '__time',
                     )}, '${start.toISOString()}/${end.toISOString()}')`,
                   ),
-                ) as SqlExpression,
+                ),
               );
               highlightStore.getState().dropHighlight();
               myChart.dispatchAction({
diff --git a/web-console/src/views/explore-view/modules/pie-chart-echarts-module.ts b/web-console/src/views/explore-view/modules/pie-chart-echarts-module.ts
index d9aedb16082..454b5f743d2 100644
--- a/web-console/src/views/explore-view/modules/pie-chart-echarts-module.ts
+++ b/web-console/src/views/explore-view/modules/pie-chart-echarts-module.ts
@@ -127,9 +127,7 @@ export default typedVisualModule({
           const others = await host.sqlQuery(
             getInitQuery(
               table,
-              where.changeClauseInWhere(
-                C(splitColumn.name).notIn(result.getColumnByIndex(0)!),
-              ) as SqlExpression,
+              where.changeClauseInWhere(C(splitColumn.name).notIn(result.getColumnByIndex(0)!)),
             ).addSelect(metric.expression.as('value')),
           );
 
diff --git a/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts b/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts
index 2e2badcfa0f..9729b5fd60d 100644
--- a/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts
+++ b/web-console/src/views/explore-view/modules/time-chart-echarts-module.ts
@@ -261,7 +261,7 @@ export default typedVisualModule({
                       timeColumnName,
                     )}, '${start.toISOString()}/${end.toISOString()}')`,
                   ),
-                ) as SqlExpression,
+                ),
               );
               highlightStore.getState().dropHighlight();
               myChart.dispatchAction({
diff --git a/web-console/src/views/workbench-view/column-tree/column-tree.spec.tsx b/web-console/src/views/workbench-view/column-tree/column-tree.spec.tsx
index 0a78a52cd2a..d17b9b1ed73 100644
--- a/web-console/src/views/workbench-view/column-tree/column-tree.spec.tsx
+++ b/web-console/src/views/workbench-view/column-tree/column-tree.spec.tsx
@@ -32,7 +32,7 @@ describe('ColumnTree', () => {
           return SqlQuery.parse(`SELECT channel, count(*) as cnt FROM wikipedia GROUP BY 1`);
         }}
         defaultSchema="druid"
-        defaultTable="wikipedia"
+        defaultTables={['wikipedia']}
         columnMetadataLoading={false}
         columnMetadata={
           [
diff --git a/web-console/src/views/workbench-view/column-tree/column-tree.tsx b/web-console/src/views/workbench-view/column-tree/column-tree.tsx
index 06a0b825b7c..12bfc19218f 100644
--- a/web-console/src/views/workbench-view/column-tree/column-tree.tsx
+++ b/web-console/src/views/workbench-view/column-tree/column-tree.tsx
@@ -167,7 +167,7 @@ export interface ColumnTreeProps {
   defaultWhere?: SqlExpression;
   onQueryChange: (query: SqlQuery, run?: boolean) => void;
   defaultSchema?: string;
-  defaultTable?: string;
+  defaultTables?: string[];
   highlightTable?: string;
 }
 
@@ -180,7 +180,7 @@ export interface ColumnTreeState {
   searchMode: SearchMode;
   prevSearchHash?: string;
   expandedTables: Map<string, boolean>;
-  prevExpandedTables: Map<string, boolean>;
+  prevExpandedTables?: Map<string, boolean>;
 }
 
 function computeSearchHash(searchString: string, searchMode: SearchMode): string {
@@ -212,14 +212,7 @@ export function getJoinColumns(parsedQuery: SqlQuery, _table: string) {
 
 export class ColumnTree extends React.PureComponent<ColumnTreeProps, ColumnTreeState> {
   static getDerivedStateFromProps(props: ColumnTreeProps, state: ColumnTreeState) {
-    const {
-      columnMetadata,
-      defaultSchema,
-      defaultTable,
-      defaultWhere,
-      onQueryChange,
-      highlightTable,
-    } = props;
+    const { columnMetadata, defaultSchema, defaultWhere, onQueryChange, highlightTable } = props;
     const { searchString, searchMode, expandedTables, prevExpandedTables } = state;
     const searchHash = computeSearchHash(searchString, searchMode);
 
@@ -227,7 +220,7 @@ export class ColumnTree extends React.PureComponent<ColumnTreeProps, ColumnTreeS
       columnMetadata &&
       (columnMetadata !== state.prevColumnMetadata ||
         searchHash !== state.prevSearchHash ||
-        expandedTables.size !== prevExpandedTables.size)
+        expandedTables !== prevExpandedTables)
     ) {
       const lowerSearchString = searchString.toLowerCase();
       const isSearching = Boolean(lowerSearchString);
@@ -569,31 +562,17 @@ export class ColumnTree extends React.PureComponent<ColumnTreeProps, ColumnTreeS
       );
 
       let selectedTreeIndex = -1;
-      let expandedNode = -1;
       if (defaultSchema && columnTree) {
         selectedTreeIndex = columnTree.findIndex(x => {
           return x.id === defaultSchema;
         });
       }
 
-      if (selectedTreeIndex > -1) {
-        const treeNodes = columnTree[selectedTreeIndex].childNodes;
-        if (treeNodes && defaultTable) {
-          expandedNode = treeNodes.findIndex(node => {
-            return node.id === defaultTable;
-          });
-        }
-      }
-
       if (!columnTree) return null;
       const currentSchemaSubtree =
         columnTree[selectedTreeIndex > -1 ? selectedTreeIndex : 0].childNodes;
       if (!currentSchemaSubtree) return null;
 
-      if (expandedNode > -1) {
-        currentSchemaSubtree[expandedNode].isExpanded = true;
-      }
-
       return {
         prevColumnMetadata: columnMetadata,
         columnTree,
@@ -612,8 +591,7 @@ export class ColumnTree extends React.PureComponent<ColumnTreeProps, ColumnTreeS
       selectedTreeIndex: -1,
       searchString: '',
       searchMode: 'tables-and-columns',
-      expandedTables: new Map(),
-      prevExpandedTables: new Map(),
+      expandedTables: new Map((props.defaultTables || []).map(t => [t, true])),
     };
   }
 
diff --git a/web-console/src/views/workbench-view/workbench-view.tsx b/web-console/src/views/workbench-view/workbench-view.tsx
index e4f15a3aa18..8da695c0be2 100644
--- a/web-console/src/views/workbench-view/workbench-view.tsx
+++ b/web-console/src/views/workbench-view/workbench-view.tsx
@@ -123,9 +123,6 @@ export interface WorkbenchViewState {
 
   details?: { id: string; initTab?: ExecutionDetailsTab; initExecution?: Execution };
 
-  defaultSchema?: string;
-  defaultTable?: string;
-
   connectExternalDataDialogOpen: boolean;
   explainDialogOpen: boolean;
   historyDialogOpen: boolean;
@@ -818,12 +815,12 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
     const { columnMetadataState, showRecentQueryTaskPanel } = this.state;
     const query = this.getCurrentQuery();
 
-    let defaultSchema;
-    let defaultTable;
+    let defaultSchema: string | undefined;
+    let defaultTables: string[] | undefined;
     const parsedQuery = query.getParsedQuery();
     if (parsedQuery) {
       defaultSchema = parsedQuery.getFirstSchema();
-      defaultTable = parsedQuery.getFirstTableName();
+      defaultTables = parsedQuery.getUsedTableNames();
     }
 
     return (
@@ -840,8 +837,8 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
             columnMetadata={columnMetadataState.data}
             onQueryChange={this.handleSqlQueryChange}
             defaultSchema={defaultSchema ? defaultSchema : 'druid'}
+            defaultTables={defaultTables}
             defaultWhere={LAST_DAY}
-            defaultTable={defaultTable}
             highlightTable={undefined}
           />
         )}

From 2198001930c01f3da7efea3c3bad57b9e94cf370 Mon Sep 17 00:00:00 2001
From: Kashif Faraz <kashif.faraz@gmail.com>
Date: Sun, 18 Aug 2024 21:45:03 -0700
Subject: [PATCH 28/99] Remove unused cachingCost strategy runtime properties
 (#16918)

---
 docs/configuration/index.md                                 | 3 +--
 .../src/main/java/org/apache/druid/cli/CliCoordinator.java  | 6 ------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 8a7e4f19981..c9ab31393d5 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -887,8 +887,7 @@ These Coordinator static configurations can be defined in the `coordinator/runti
 |`druid.coordinator.kill.ignoreDurationToRetain`|A way to override `druid.coordinator.kill.durationToRetain` and tell the coordinator that you do not care about the end date of unused segment intervals when it comes to killing them. If true, the coordinator considers all unused segments as eligible to be killed.|false|
 |`druid.coordinator.kill.bufferPeriod`|The amount of time that a segment must be unused before it is able to be permanently removed from metadata and deep storage. This can serve as a buffer period to prevent data loss if data ends up being needed after being marked unused.|`P30D`|
 |`druid.coordinator.kill.maxSegments`|The number of unused segments to kill per kill task. This number must be greater than 0. This only applies when `druid.coordinator.kill.on=true`.|100|
-|`druid.coordinator.balancer.strategy`|Specify the type of balancing strategy for the Coordinator to use to distribute segments among the Historical services. `cachingCost` is logically equivalent to `cost` but is more CPU-efficient on large clusters. `diskNormalized` weights the costs according to the servers' disk usage ratios - there are known issues with this strategy distributing segments unevenly across the cluster. `random` distributes segments among services randomly.|`cost`|
-|`druid.coordinator.balancer.cachingCost.awaitInitialization`|Whether to wait for segment view initialization before creating the `cachingCost` balancing strategy. This property is enabled only when `druid.coordinator.balancer.strategy` is `cachingCost`. If set to true, the Coordinator will not start to assign segments, until the segment view is initialized. If set to false, the Coordinator will fallback to use the `cost` balancing strategy only if the segment view is not initialized yet. It may take much time to wait for the initialization since the `cachingCost` balancing strategy involves much computing to build itself.|false|
+|`druid.coordinator.balancer.strategy`|Specify the type of balancing strategy for the Coordinator to use to distribute segments among the Historical services. `diskNormalized` weights the costs according to the servers' disk usage ratios - there are known issues with this strategy distributing segments unevenly across the cluster. `random` distributes segments among services randomly.|`cost`|
 |`druid.coordinator.loadqueuepeon.http.repeatDelay`|The start and repeat delay (in milliseconds) for the load queue peon, which manages the load/drop queue of segments for any server.|1 minute|
 |`druid.coordinator.loadqueuepeon.http.batchSize`|Number of segment load/drop requests to batch in one HTTP request. Note that it must be smaller than `druid.segmentCache.numLoadingThreads` config on Historical service.|1|
 |`druid.coordinator.asOverlord.enabled`|Boolean value for whether this Coordinator service should act like an Overlord as well. This configuration allows users to simplify a Druid cluster by not having to deploy any standalone Overlord services. If set to true, then Overlord console is available at `http://coordinator-host:port/console.html` and be sure to set `druid.coordinator.asOverlord.overlordService` also.|false|
diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
index 7adaac5b88d..9b8c758a745 100644
--- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
+++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
@@ -101,7 +101,6 @@ import org.apache.druid.server.coordinator.CoordinatorConfigManager;
 import org.apache.druid.server.coordinator.DruidCoordinator;
 import org.apache.druid.server.coordinator.MetadataManager;
 import org.apache.druid.server.coordinator.balancer.BalancerStrategyFactory;
-import org.apache.druid.server.coordinator.balancer.CachingCostBalancerStrategyConfig;
 import org.apache.druid.server.coordinator.compact.CompactionSegmentSearchPolicy;
 import org.apache.druid.server.coordinator.compact.NewestSegmentFirstPolicy;
 import org.apache.druid.server.coordinator.config.CoordinatorKillConfigs;
@@ -223,11 +222,6 @@ public class CliCoordinator extends ServerRunnable
             JsonConfigProvider.bind(binder, "druid.coordinator.loadqueuepeon.http", HttpLoadQueuePeonConfig.class);
             JsonConfigProvider.bind(binder, "druid.coordinator.balancer", BalancerStrategyFactory.class);
             JsonConfigProvider.bind(binder, "druid.coordinator.segment", CoordinatorSegmentWatcherConfig.class);
-            JsonConfigProvider.bind(
-                binder,
-                "druid.coordinator.balancer.cachingCost",
-                CachingCostBalancerStrategyConfig.class
-            );
             JsonConfigProvider.bind(binder, "druid.coordinator.segmentMetadataCache", SegmentMetadataCacheConfig.class);
             binder.bind(DruidCoordinatorConfig.class);
 

From fb7103cceff52944c29ca365e56b3ebb23d2e5be Mon Sep 17 00:00:00 2001
From: Vishesh Garg <gargvishesh@gmail.com>
Date: Tue, 20 Aug 2024 10:30:24 +0530
Subject: [PATCH 29/99] Change dimensionToSchemaMap to dimensionSchemas and
 override ARRAY_INGEST_MODE to array (#16909)

A follow-up PR for #16864. Just renames dimensionToSchemaMap to dimensionSchemas and always overrides ARRAY_INGEST_MODE context value to array for MSQ compaction.
---
 .../apache/druid/msq/exec/ControllerImpl.java | 20 +++++++++----------
 .../msq/indexing/MSQCompactionRunner.java     |  3 ++-
 .../destination/DataSourceMSQDestination.java | 16 +++++++--------
 .../DataSourceMSQDestinationTest.java         |  2 +-
 4 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
index 278a85685dd..4310619f81f 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
@@ -1951,7 +1951,7 @@ public class ControllerImpl implements Controller
             columnMappings,
             isRollupQuery,
             querySpec.getQuery(),
-            destination.getDimensionToSchemaMap()
+            destination.getDimensionSchemas()
         );
 
     return new DataSchema(
@@ -2127,11 +2127,11 @@ public class ControllerImpl implements Controller
       final String outputColumnName,
       @Nullable final ColumnType queryType,
       QueryContext context,
-      @Nullable Map<String, DimensionSchema> dimensionToSchemaMap
+      @Nullable Map<String, DimensionSchema> dimensionSchemas
   )
   {
-    if (dimensionToSchemaMap != null && dimensionToSchemaMap.containsKey(outputColumnName)) {
-      return dimensionToSchemaMap.get(outputColumnName);
+    if (dimensionSchemas != null && dimensionSchemas.containsKey(outputColumnName)) {
+      return dimensionSchemas.get(outputColumnName);
     }
     // In case of ingestion, or when metrics are converted to dimensions when compaction is performed without rollup,
     // we won't have an entry in the map. For those cases, use the default config.
@@ -2150,7 +2150,7 @@ public class ControllerImpl implements Controller
       final ColumnMappings columnMappings,
       final boolean isRollupQuery,
       final Query<?> query,
-      @Nullable final Map<String, DimensionSchema> dimensionToSchemaMap
+      @Nullable final Map<String, DimensionSchema> dimensionSchemas
   )
   {
     // Log a warning unconditionally if arrayIngestMode is MVD, since the behaviour is incorrect, and is subject to
@@ -2237,13 +2237,13 @@ public class ControllerImpl implements Controller
               outputColumnName,
               type,
               query.context(),
-              dimensionToSchemaMap
+              dimensionSchemas
           );
         } else {
           // complex columns only
           if (DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.containsKey(type.getComplexTypeName())) {
             dimensions.add(
-                getDimensionSchema(outputColumnName, type, query.context(), dimensionToSchemaMap)
+                getDimensionSchema(outputColumnName, type, query.context(), dimensionSchemas)
             );
           } else if (!isRollupQuery) {
             aggregators.add(new PassthroughAggregatorFactory(outputColumnName, type.getComplexTypeName()));
@@ -2255,7 +2255,7 @@ public class ControllerImpl implements Controller
                 outputColumnName,
                 type,
                 query.context(),
-                dimensionToSchemaMap
+                dimensionSchemas
             );
           }
         }
@@ -2283,14 +2283,14 @@ public class ControllerImpl implements Controller
       String outputColumn,
       ColumnType type,
       QueryContext context,
-      Map<String, DimensionSchema> dimensionToSchemaMap
+      Map<String, DimensionSchema> dimensionSchemas
   )
   {
     if (outputColumnAggregatorFactories.containsKey(outputColumn)) {
       aggregators.add(outputColumnAggregatorFactories.get(outputColumn));
     } else {
       dimensions.add(
-          getDimensionSchema(outputColumn, type, context, dimensionToSchemaMap)
+          getDimensionSchema(outputColumn, type, context, dimensionSchemas)
       );
     }
   }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
index aeb91a6c253..fa011429763 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
@@ -503,7 +503,8 @@ public class MSQCompactionRunner implements CompactionRunner
     context.putIfAbsent(QueryContexts.FINALIZE_KEY, false);
     // Only scalar or array-type dimensions are allowed as grouping keys.
     context.putIfAbsent(GroupByQueryConfig.CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING, false);
-    context.putIfAbsent(MultiStageQueryContext.CTX_ARRAY_INGEST_MODE, "array");
+    // Always override CTX_ARRAY_INGEST_MODE since it can otherwise lead to mixed ARRAY and MVD types for a column.
+    context.put(MultiStageQueryContext.CTX_ARRAY_INGEST_MODE, "array");
     return context;
   }
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
index 74be1329467..6276c588d9d 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
@@ -52,7 +52,7 @@ public class DataSourceMSQDestination implements MSQDestination
   private final List<Interval> replaceTimeChunks;
 
   @Nullable
-  private final Map<String, DimensionSchema> dimensionToSchemaMap;
+  private final Map<String, DimensionSchema> dimensionSchemas;
 
   @JsonCreator
   public DataSourceMSQDestination(
@@ -60,14 +60,14 @@ public class DataSourceMSQDestination implements MSQDestination
       @JsonProperty("segmentGranularity") Granularity segmentGranularity,
       @JsonProperty("segmentSortOrder") @Nullable List<String> segmentSortOrder,
       @JsonProperty("replaceTimeChunks") @Nullable List<Interval> replaceTimeChunks,
-      @JsonProperty("dimensionToSchemaMap") @Nullable Map<String, DimensionSchema> dimensionToSchemaMap
+      @JsonProperty("dimensionSchemas") @Nullable Map<String, DimensionSchema> dimensionSchemas
   )
   {
     this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource");
     this.segmentGranularity = Preconditions.checkNotNull(segmentGranularity, "segmentGranularity");
     this.segmentSortOrder = segmentSortOrder != null ? segmentSortOrder : Collections.emptyList();
     this.replaceTimeChunks = replaceTimeChunks;
-    this.dimensionToSchemaMap = dimensionToSchemaMap;
+    this.dimensionSchemas = dimensionSchemas;
 
     if (replaceTimeChunks != null) {
       // Verify that if replaceTimeChunks is provided, it is nonempty.
@@ -138,9 +138,9 @@ public class DataSourceMSQDestination implements MSQDestination
   @Nullable
   @JsonProperty
   @JsonInclude(JsonInclude.Include.NON_NULL)
-  public Map<String, DimensionSchema> getDimensionToSchemaMap()
+  public Map<String, DimensionSchema> getDimensionSchemas()
   {
-    return dimensionToSchemaMap;
+    return dimensionSchemas;
   }
 
   /**
@@ -177,13 +177,13 @@ public class DataSourceMSQDestination implements MSQDestination
            && Objects.equals(segmentGranularity, that.segmentGranularity)
            && Objects.equals(segmentSortOrder, that.segmentSortOrder)
            && Objects.equals(replaceTimeChunks, that.replaceTimeChunks)
-           && Objects.equals(dimensionToSchemaMap, that.dimensionToSchemaMap);
+           && Objects.equals(dimensionSchemas, that.dimensionSchemas);
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(dataSource, segmentGranularity, segmentSortOrder, replaceTimeChunks, dimensionToSchemaMap);
+    return Objects.hash(dataSource, segmentGranularity, segmentSortOrder, replaceTimeChunks, dimensionSchemas);
   }
 
   @Override
@@ -194,7 +194,7 @@ public class DataSourceMSQDestination implements MSQDestination
            ", segmentGranularity=" + segmentGranularity +
            ", segmentSortOrder=" + segmentSortOrder +
            ", replaceTimeChunks=" + replaceTimeChunks +
-           ", dimensionToSchemaMap=" + dimensionToSchemaMap +
+           ", dimensionSchemas=" + dimensionSchemas +
            '}';
   }
 
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
index 242c00213e2..331430181a9 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
@@ -35,7 +35,7 @@ public class DataSourceMSQDestinationTest
   public void testEquals()
   {
     EqualsVerifier.forClass(DataSourceMSQDestination.class)
-                  .withNonnullFields("dataSource", "segmentGranularity", "segmentSortOrder", "dimensionToSchemaMap")
+                  .withNonnullFields("dataSource", "segmentGranularity", "segmentSortOrder", "dimensionSchemas")
                   .withPrefabValues(
                       Map.class,
                       ImmutableMap.of(

From 518f642028654befbaa09c328b7ccaf30ffd6f63 Mon Sep 17 00:00:00 2001
From: Clint Wylie <cwylie@apache.org>
Date: Mon, 19 Aug 2024 23:02:45 -0700
Subject: [PATCH 30/99] remove isDescending from Query interface, move to
 TimeseriesQuery (#16917)

* remove isDescending from Query interface, since it is only actually settable and usable by TimeseriesQuery
---
 .../MaterializedViewQuery.java                |  6 ------
 .../org/apache/druid/query/BaseQuery.java     | 21 ++++---------------
 .../java/org/apache/druid/query/Query.java    |  2 --
 .../druid/query/groupby/GroupByQuery.java     |  2 +-
 .../druid/query/search/SearchQuery.java       |  2 +-
 .../druid/query/select/SelectQuery.java       |  6 ------
 .../query/timeseries/TimeseriesQuery.java     | 20 +++++++++++++++++-
 .../TimeseriesQueryQueryToolChest.java        |  2 +-
 .../apache/druid/query/topn/TopNQuery.java    |  2 +-
 .../query/topn/TopNQueryQueryToolChest.java   |  2 +-
 .../apache/druid/query/QueryContextTest.java  |  6 ------
 .../server/log/LoggingRequestLogger.java      |  1 -
 .../server/log/LoggingRequestLoggerTest.java  |  5 -----
 13 files changed, 28 insertions(+), 49 deletions(-)

diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQuery.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQuery.java
index d771667f6ea..ec061370a4d 100644
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQuery.java
+++ b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQuery.java
@@ -146,12 +146,6 @@ public class MaterializedViewQuery<T> implements Query<T>
     return query.getContext();
   }
 
-  @Override
-  public boolean isDescending()
-  {
-    return query.isDescending();
-  }
-
   @Override
   public Ordering<T> getResultOrdering()
   {
diff --git a/processing/src/main/java/org/apache/druid/query/BaseQuery.java b/processing/src/main/java/org/apache/druid/query/BaseQuery.java
index f25a5ef317e..bbfa570e309 100644
--- a/processing/src/main/java/org/apache/druid/query/BaseQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/BaseQuery.java
@@ -57,7 +57,6 @@ public abstract class BaseQuery<T> implements Query<T>
   public static final String SUB_QUERY_ID = "subQueryId";
   public static final String SQL_QUERY_ID = "sqlQueryId";
   private final DataSource dataSource;
-  private final boolean descending;
   private final QueryContext context;
   private final QuerySegmentSpec querySegmentSpec;
   private volatile Duration duration;
@@ -69,13 +68,12 @@ public abstract class BaseQuery<T> implements Query<T>
       Map<String, Object> context
   )
   {
-    this(dataSource, querySegmentSpec, false, context, Granularities.ALL);
+    this(dataSource, querySegmentSpec, context, Granularities.ALL);
   }
 
   public BaseQuery(
       DataSource dataSource,
       QuerySegmentSpec querySegmentSpec,
-      boolean descending,
       Map<String, Object> context,
       Granularity granularity
   )
@@ -87,7 +85,6 @@ public abstract class BaseQuery<T> implements Query<T>
     this.dataSource = dataSource;
     this.context = QueryContext.of(context);
     this.querySegmentSpec = querySegmentSpec;
-    this.descending = descending;
     this.granularity = granularity;
   }
 
@@ -98,14 +95,6 @@ public abstract class BaseQuery<T> implements Query<T>
     return dataSource;
   }
 
-  @JsonProperty
-  @Override
-  @JsonInclude(Include.NON_DEFAULT)
-  public boolean isDescending()
-  {
-    return descending;
-  }
-
   @JsonProperty("intervals")
   public QuerySegmentSpec getQuerySegmentSpec()
   {
@@ -205,8 +194,7 @@ public abstract class BaseQuery<T> implements Query<T>
   @SuppressWarnings("unchecked") // assumes T is Comparable; see method javadoc
   public Ordering<T> getResultOrdering()
   {
-    Ordering retVal = Ordering.natural();
-    return descending ? retVal.reverse() : retVal;
+    return (Ordering<T>) Ordering.natural();
   }
 
   @Nullable
@@ -253,8 +241,7 @@ public abstract class BaseQuery<T> implements Query<T>
     BaseQuery<?> baseQuery = (BaseQuery<?>) o;
 
     // Must use getDuration() instead of "duration" because duration is lazily computed.
-    return descending == baseQuery.descending &&
-           Objects.equals(dataSource, baseQuery.dataSource) &&
+    return Objects.equals(dataSource, baseQuery.dataSource) &&
            Objects.equals(context, baseQuery.context) &&
            Objects.equals(querySegmentSpec, baseQuery.querySegmentSpec) &&
            Objects.equals(getDuration(), baseQuery.getDuration()) &&
@@ -265,6 +252,6 @@ public abstract class BaseQuery<T> implements Query<T>
   public int hashCode()
   {
     // Must use getDuration() instead of "duration" because duration is lazily computed.
-    return Objects.hash(dataSource, descending, context, querySegmentSpec, getDuration(), granularity);
+    return Objects.hash(dataSource, context, querySegmentSpec, getDuration(), granularity);
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java
index fa1f42d4c13..4795ecbb561 100644
--- a/processing/src/main/java/org/apache/druid/query/Query.java
+++ b/processing/src/main/java/org/apache/druid/query/Query.java
@@ -166,8 +166,6 @@ public interface Query<T>
     return context().getHumanReadableBytes(key, defaultValue);
   }
 
-  boolean isDescending();
-
   /**
    * Comparator that represents the order in which results are generated from the
    * {@link QueryRunnerFactory#createRunner(Segment)} and
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
index 761aa43b1b3..35510d7c5ee 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
@@ -202,7 +202,7 @@ public class GroupByQuery extends BaseQuery<ResultRow>
       final Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec, false, context, granularity);
+    super(dataSource, querySegmentSpec, context, granularity);
 
     this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns);
     this.dimFilter = dimFilter;
diff --git a/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java b/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java
index 7424bbf1fbc..aa10b6ab579 100644
--- a/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java
@@ -68,7 +68,7 @@ public class SearchQuery extends BaseQuery<Result<SearchResultValue>>
       @JsonProperty("context") Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec, false, context, Granularities.nullToAll(granularity));
+    super(dataSource, querySegmentSpec, context, Granularities.nullToAll(granularity));
     Preconditions.checkNotNull(querySegmentSpec, "Must specify an interval");
 
     this.dimFilter = dimFilter;
diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectQuery.java b/processing/src/main/java/org/apache/druid/query/select/SelectQuery.java
index 606ccbeb357..c24f6f344bd 100644
--- a/processing/src/main/java/org/apache/druid/query/select/SelectQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/select/SelectQuery.java
@@ -110,12 +110,6 @@ public class SelectQuery implements Query<Object>
     throw new RuntimeException(REMOVED_ERROR_MESSAGE);
   }
 
-  @Override
-  public boolean isDescending()
-  {
-    throw new RuntimeException(REMOVED_ERROR_MESSAGE);
-  }
-
   @Override
   public Ordering<Object> getResultOrdering()
   {
diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java
index 85ddf5aaa62..6e2cb62adcf 100644
--- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Ordering;
 import org.apache.commons.lang.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.query.BaseQuery;
@@ -68,6 +69,7 @@ public class TimeseriesQuery extends BaseQuery<Result<TimeseriesResultValue>>
   private final DimFilter dimFilter;
   private final List<AggregatorFactory> aggregatorSpecs;
   private final List<PostAggregator> postAggregatorSpecs;
+  private final boolean descending;
   private final int limit;
 
   @JsonCreator
@@ -84,7 +86,7 @@ public class TimeseriesQuery extends BaseQuery<Result<TimeseriesResultValue>>
       @JsonProperty("context") Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec, descending, context, granularity);
+    super(dataSource, querySegmentSpec, context, granularity);
 
     // The below should be executed after context is initialized.
     final String timestampField = getTimestampResultField();
@@ -97,6 +99,7 @@ public class TimeseriesQuery extends BaseQuery<Result<TimeseriesResultValue>>
         this.aggregatorSpecs,
         postAggregatorSpecs == null ? ImmutableList.of() : postAggregatorSpecs
     );
+    this.descending = descending;
     this.limit = (limit == 0) ? Integer.MAX_VALUE : limit;
     Preconditions.checkArgument(this.limit > 0, "limit must be greater than 0");
   }
@@ -148,6 +151,13 @@ public class TimeseriesQuery extends BaseQuery<Result<TimeseriesResultValue>>
     return postAggregatorSpecs;
   }
 
+  @JsonProperty
+  @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+  public boolean isDescending()
+  {
+    return descending;
+  }
+
   @JsonProperty("limit")
   @JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = LimitJsonIncludeFilter.class)
   public int getLimit()
@@ -155,6 +165,7 @@ public class TimeseriesQuery extends BaseQuery<Result<TimeseriesResultValue>>
     return limit;
   }
 
+
   public boolean isGrandTotal()
   {
     return context().getBoolean(CTX_GRAND_TOTAL, false);
@@ -195,6 +206,13 @@ public class TimeseriesQuery extends BaseQuery<Result<TimeseriesResultValue>>
     );
   }
 
+  @Override
+  public Ordering<Result<TimeseriesResultValue>> getResultOrdering()
+  {
+    Ordering<Result<TimeseriesResultValue>> retVal = Ordering.natural();
+    return descending ? retVal.reverse() : retVal;
+  }
+
   @Override
   public TimeseriesQuery withQuerySegmentSpec(QuerySegmentSpec querySegmentSpec)
   {
diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java
index 3ad58270f8b..05b25d05c21 100644
--- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java
+++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java
@@ -228,7 +228,7 @@ public class TimeseriesQueryQueryToolChest extends QueryToolChest<Result<Timeser
   @Override
   public Comparator<Result<TimeseriesResultValue>> createResultComparator(Query<Result<TimeseriesResultValue>> query)
   {
-    return ResultGranularTimestampComparator.create(query.getGranularity(), query.isDescending());
+    return ResultGranularTimestampComparator.create(query.getGranularity(), ((TimeseriesQuery) query).isDescending());
   }
 
   private Result<TimeseriesResultValue> getNullTimeseriesResultValue(TimeseriesQuery query)
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java
index 2534c37f90f..3178ac15a18 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java
@@ -76,7 +76,7 @@ public class TopNQuery extends BaseQuery<Result<TopNResultValue>>
       @JsonProperty("context") Map<String, Object> context
   )
   {
-    super(dataSource, querySegmentSpec, false, context, granularity);
+    super(dataSource, querySegmentSpec, context, granularity);
 
     Preconditions.checkNotNull(dimensionSpec, "dimensionSpec can't be null");
     Preconditions.checkNotNull(topNMetricSpec, "must specify a metric");
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryQueryToolChest.java
index c5f195615f3..501684dce40 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryQueryToolChest.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryQueryToolChest.java
@@ -124,7 +124,7 @@ public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultVal
   {
     final ResultMergeQueryRunner<Result<TopNResultValue>> delegateRunner = new ResultMergeQueryRunner<>(
         runner,
-        query -> ResultGranularTimestampComparator.create(query.getGranularity(), query.isDescending()),
+        query -> ResultGranularTimestampComparator.create(query.getGranularity(), false),
         query -> {
           TopNQuery topNQuery = (TopNQuery) query;
           return new TopNBinaryFn(
diff --git a/processing/src/test/java/org/apache/druid/query/QueryContextTest.java b/processing/src/test/java/org/apache/druid/query/QueryContextTest.java
index c555c2ed437..d0d6a512627 100644
--- a/processing/src/test/java/org/apache/druid/query/QueryContextTest.java
+++ b/processing/src/test/java/org/apache/druid/query/QueryContextTest.java
@@ -464,12 +464,6 @@ public class QueryContextTest
       return context;
     }
 
-    @Override
-    public boolean isDescending()
-    {
-      return false;
-    }
-
     @Override
     public Ordering<Integer> getResultOrdering()
     {
diff --git a/server/src/main/java/org/apache/druid/server/log/LoggingRequestLogger.java b/server/src/main/java/org/apache/druid/server/log/LoggingRequestLogger.java
index 4a8b946eecc..08e8e92d739 100644
--- a/server/src/main/java/org/apache/druid/server/log/LoggingRequestLogger.java
+++ b/server/src/main/java/org/apache/druid/server/log/LoggingRequestLogger.java
@@ -68,7 +68,6 @@ public class LoggingRequestLogger implements RequestLogger
           MDC.put("hasFilters", Boolean.toString(query.hasFilters()));
           MDC.put("remoteAddr", requestLogLine.getRemoteAddr());
           MDC.put("duration", query.getDuration().toString());
-          MDC.put("descending", Boolean.toString(query.isDescending()));
           if (setContextMDC) {
             final Iterable<Map.Entry<String, Object>> entries = query.getContext() == null
                                                                 ? ImmutableList.of()
diff --git a/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java b/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java
index 0ee15ab1f3f..3dc9696d4f4 100644
--- a/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java
+++ b/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java
@@ -219,7 +219,6 @@ public class LoggingRequestLoggerTest
     Assert.assertEquals("false", map.get("hasFilters"));
     Assert.assertEquals("fake", map.get("queryType"));
     Assert.assertEquals("some.host.tld", map.get("remoteAddr"));
-    Assert.assertEquals("false", map.get("descending"));
     Assert.assertEquals("false", map.get("isNested"));
     Assert.assertNull(map.get("foo"));
   }
@@ -235,7 +234,6 @@ public class LoggingRequestLoggerTest
     Assert.assertEquals("false", map.get("hasFilters"));
     Assert.assertEquals("fake", map.get("queryType"));
     Assert.assertEquals("some.host.tld", map.get("remoteAddr"));
-    Assert.assertEquals("false", map.get("descending"));
     Assert.assertEquals("false", map.get("isNested"));
     Assert.assertEquals("bar", map.get("foo"));
   }
@@ -256,7 +254,6 @@ public class LoggingRequestLoggerTest
     Assert.assertEquals("false", map.get("hasFilters"));
     Assert.assertEquals("fake", map.get("queryType"));
     Assert.assertEquals("some.host.tld", map.get("remoteAddr"));
-    Assert.assertEquals("false", map.get("descending"));
     Assert.assertEquals("true", map.get("isNested"));
     Assert.assertNull(map.get("foo"));
   }
@@ -278,7 +275,6 @@ public class LoggingRequestLoggerTest
     Assert.assertEquals("fake", map.get("queryType"));
     Assert.assertEquals("some.host.tld", map.get("remoteAddr"));
     Assert.assertEquals("true", map.get("isNested"));
-    Assert.assertEquals("false", map.get("descending"));
     Assert.assertNull(map.get("foo"));
   }
 
@@ -299,7 +295,6 @@ public class LoggingRequestLoggerTest
     Assert.assertEquals("false", map.get("hasFilters"));
     Assert.assertEquals("fake", map.get("queryType"));
     Assert.assertEquals("some.host.tld", map.get("remoteAddr"));
-    Assert.assertEquals("false", map.get("descending"));
     Assert.assertNull(map.get("foo"));
   }
 

From bc4b3a2f91df258fe84573182b465ed8b74b92cb Mon Sep 17 00:00:00 2001
From: Rishabh Singh <6513075+findingrish@users.noreply.github.com>
Date: Tue, 20 Aug 2024 11:35:02 +0530
Subject: [PATCH 31/99] Filter out tombstone segments from metadata cache
 (#16890)

* Fix build

* Support segment metadata queries for tombstones

* Filter out tombstone segments from metadata cache

* Revert some changes

* checkstyle

* Update docs
---
 .../AbstractSegmentMetadataCache.java         | 11 +++
 .../CoordinatorSegmentMetadataCacheTest.java  | 72 +++++++++++++++++++
 .../BrokerSegmentMetadataCacheTest.java       | 70 ++++++++++++++++++
 3 files changed, 153 insertions(+)

diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
index 88e6ee97b98..06ce009c8b2 100644
--- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
@@ -458,6 +458,13 @@ public abstract class AbstractSegmentMetadataCache<T extends DataSourceInformati
   @VisibleForTesting
   public void addSegment(final DruidServerMetadata server, final DataSegment segment)
   {
+    // Skip adding tombstone segment to the cache. These segments lack data or column information.
+    // Additionally, segment metadata queries, which are not yet implemented for tombstone segments
+    // (see: https://github.com/apache/druid/pull/12137) do not provide metadata for tombstones,
+    // leading to indefinite refresh attempts for these segments.
+    if (segment.isTombstone()) {
+      return;
+    }
     // Get lock first so that we won't wait in ConcurrentMap.compute().
     synchronized (lock) {
       // someday we could hypothetically remove broker special casing, whenever BrokerServerView supports tracking
@@ -530,6 +537,10 @@ public abstract class AbstractSegmentMetadataCache<T extends DataSourceInformati
   @VisibleForTesting
   public void removeSegment(final DataSegment segment)
   {
+    // tombstone segments are not present in the cache
+    if (segment.isTombstone()) {
+      return;
+    }
     // Get lock first so that we won't wait in ConcurrentMap.compute().
     synchronized (lock) {
       log.debug("Segment [%s] is gone.", segment.getId());
diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
index 8fbc78a7412..772a79ae0ad 100644
--- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
+++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
@@ -81,6 +81,7 @@ import org.apache.druid.server.security.NoopEscalator;
 import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.SegmentId;
 import org.apache.druid.timeline.partition.LinearShardSpec;
+import org.apache.druid.timeline.partition.TombstoneShardSpec;
 import org.easymock.EasyMock;
 import org.joda.time.Period;
 import org.junit.After;
@@ -2214,6 +2215,77 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
     Assert.assertEquals(0, latch.getCount());
   }
 
+  @Test
+  public void testTombstoneSegmentIsNotAdded() throws InterruptedException
+  {
+    String datasource = "newSegmentAddTest";
+    CountDownLatch addSegmentLatch = new CountDownLatch(1);
+
+    CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache(
+        getQueryLifecycleFactory(walker),
+        serverView,
+        SEGMENT_CACHE_CONFIG_DEFAULT,
+        new NoopEscalator(),
+        new InternalQueryConfig(),
+        new NoopServiceEmitter(),
+        segmentSchemaCache,
+        backFillQueue,
+        sqlSegmentsMetadataManager,
+        segmentsMetadataManagerConfigSupplier
+    )
+    {
+      @Override
+      public void addSegment(final DruidServerMetadata server, final DataSegment segment)
+      {
+        super.addSegment(server, segment);
+        if (datasource.equals(segment.getDataSource())) {
+          addSegmentLatch.countDown();
+        }
+      }
+    };
+
+    schema.onLeaderStart();
+    schema.awaitInitialization();
+
+    DataSegment segment = new DataSegment(
+        datasource,
+        Intervals.of("2001/2002"),
+        "1",
+        Collections.emptyMap(),
+        Collections.emptyList(),
+        Collections.emptyList(),
+        TombstoneShardSpec.INSTANCE,
+        null,
+        null,
+        0
+    );
+
+    Assert.assertEquals(6, schema.getTotalSegments());
+
+    serverView.addSegment(segment, ServerType.HISTORICAL);
+    Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS));
+    Assert.assertEquals(0, addSegmentLatch.getCount());
+
+    Assert.assertEquals(6, schema.getTotalSegments());
+    List<AvailableSegmentMetadata> metadatas = schema
+        .getSegmentMetadataSnapshot()
+        .values()
+        .stream()
+        .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource()))
+        .collect(Collectors.toList());
+    Assert.assertEquals(0, metadatas.size());
+
+    serverView.removeSegment(segment, ServerType.HISTORICAL);
+    Assert.assertEquals(6, schema.getTotalSegments());
+    metadatas = schema
+        .getSegmentMetadataSnapshot()
+        .values()
+        .stream()
+        .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource()))
+        .collect(Collectors.toList());
+    Assert.assertEquals(0, metadatas.size());
+  }
+
   private void verifyFooDSSchema(CoordinatorSegmentMetadataCache schema, int columns)
   {
     final DataSourceInformation fooDs = schema.getDatasource("foo");
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java
index 65610ce99f2..10504ca49d5 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java
@@ -79,6 +79,7 @@ import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.SegmentId;
 import org.apache.druid.timeline.partition.LinearShardSpec;
 import org.apache.druid.timeline.partition.NumberedShardSpec;
+import org.apache.druid.timeline.partition.TombstoneShardSpec;
 import org.easymock.EasyMock;
 import org.joda.time.Period;
 import org.junit.After;
@@ -1136,4 +1137,73 @@ public class BrokerSegmentMetadataCacheTest extends BrokerSegmentMetadataCacheTe
 
     Assert.assertNull(schema.getDatasource("foo"));
   }
+
+  @Test
+  public void testTombstoneSegmentIsNotAdded() throws InterruptedException
+  {
+    String datasource = "newSegmentAddTest";
+    CountDownLatch addSegmentLatch = new CountDownLatch(1);
+    BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache(
+        CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
+        serverView,
+        BrokerSegmentMetadataCacheConfig.create(),
+        new NoopEscalator(),
+        new InternalQueryConfig(),
+        new NoopServiceEmitter(),
+        new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager),
+        new NoopCoordinatorClient(),
+        CentralizedDatasourceSchemaConfig.create()
+    )
+    {
+      @Override
+      public void addSegment(final DruidServerMetadata server, final DataSegment segment)
+      {
+        super.addSegment(server, segment);
+        if (datasource.equals(segment.getDataSource())) {
+          addSegmentLatch.countDown();
+        }
+      }
+    };
+
+    schema.start();
+    schema.awaitInitialization();
+
+    DataSegment segment = new DataSegment(
+        datasource,
+        Intervals.of("2001/2002"),
+        "1",
+        Collections.emptyMap(),
+        Collections.emptyList(),
+        Collections.emptyList(),
+        TombstoneShardSpec.INSTANCE,
+        null,
+        null,
+        0
+    );
+
+    Assert.assertEquals(6, schema.getTotalSegments());
+
+    serverView.addSegment(segment, ServerType.HISTORICAL);
+    Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS));
+    Assert.assertEquals(0, addSegmentLatch.getCount());
+
+    Assert.assertEquals(6, schema.getTotalSegments());
+    List<AvailableSegmentMetadata> metadatas = schema
+        .getSegmentMetadataSnapshot()
+        .values()
+        .stream()
+        .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource()))
+        .collect(Collectors.toList());
+    Assert.assertEquals(0, metadatas.size());
+
+    serverView.removeSegment(segment, ServerType.HISTORICAL);
+    Assert.assertEquals(6, schema.getTotalSegments());
+    metadatas = schema
+        .getSegmentMetadataSnapshot()
+        .values()
+        .stream()
+        .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource()))
+        .collect(Collectors.toList());
+    Assert.assertEquals(0, metadatas.size());
+  }
 }

From 82d9ff9cc8feba8280a694254970eb67fa8fd35c Mon Sep 17 00:00:00 2001
From: Jakub Matyszewski <jakubmatyszewski1@gmail.com>
Date: Tue, 20 Aug 2024 12:28:31 +0200
Subject: [PATCH 32/99] Add docs for log audit manager (#16927)

* Add docs for log audit manager

* Adjust descriptions
---
 docs/configuration/index.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index c9ab31393d5..61ddd8abcea 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -363,6 +363,8 @@ Coordinator and Overlord log changes to lookups, segment load/drop rules, and dy
 
 |Property|Description|Default|
 |--------|-----------|-------|
+|`druid.audit.manager.type`|Type of audit manager used for handling audited events. Audited events are logged when set to `log` or persisted in metadata store when set to `sql`.|sql|
+|`druid.audit.manager.logLevel`|Log level of audit events with possible values DEBUG, INFO, WARN. This property is used only when `druid.audit.manager.type` is set to `log`.|INFO|
 |`druid.audit.manager.auditHistoryMillis`|Default duration for querying audit history.|1 week|
 |`druid.audit.manager.includePayloadAsDimensionInMetric`|Boolean flag on whether to add `payload` column in service metric.|false|
 |`druid.audit.manager.maxPayloadSizeBytes`|The maximum size of audit payload to store in Druid's metadata store audit table. If the size of audit payload exceeds this value, the audit log would be stored with a message indicating that the payload was omitted instead. Setting `maxPayloadSizeBytes` to -1 (default value) disables this check, meaning Druid will always store audit payload regardless of it's size. Setting to any negative number other than `-1` is invalid. Human-readable format is supported, see [here](human-readable-byte.md).  |-1|

From 7b8573ed3d16efb8fc5ff3c59985fddb256f06e8 Mon Sep 17 00:00:00 2001
From: benkrug <benkrug@yahoo.com>
Date: Tue, 20 Aug 2024 11:06:12 -0700
Subject: [PATCH 33/99] Update index.md - remove the extra word "does" from one
 sentence. (#16922)

Co-authored-by: 317brian <53799971+317brian@users.noreply.github.com>
---
 docs/configuration/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 61ddd8abcea..637ebeabc0e 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -1529,7 +1529,7 @@ The following types of medium exist for the buffers:
 This type of medium may do unnecessary disk I/O and requires some disk space to be available.
 
 * **Off-heap memory** (`offHeapMemory`) creates buffers in off-heap memory of a JVM process that is running a task.
-This type of medium is preferred, but it may require to allow the JVM to have more off-heap memory, by changing `-XX:MaxDirectMemorySize` configuration. It is not yet understood how does the required off-heap memory size relates to the size of the segments being created. But definitely it doesn't make sense to add more extra off-heap memory, than the configured maximum _heap_ size (`-Xmx`) for the same JVM.
+This type of medium is preferred, but it may require you to allow the JVM to have more off-heap memory by changing the `-XX:MaxDirectMemorySize` configuration. It's not understood yet how the required off-heap memory size relates to the size of the segments being created. But you shouldn't add more extra off-heap memory than the configured maximum _heap_ size (`-Xmx`) for the same JVM.
 
 * **On-heap memory** (`onHeapMemory`) creates buffers using the allocated heap memory of the JVM process running a task. Using on-heap memory introduces garbage collection overhead and so is not recommended in most cases. This type of medium is most helpful for tasks run on external clusters where it may be difficult to allocate and work with direct memory effectively.
 

From 2bd31603de847e31a01d6cf6fa16fb127947db56 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Tue, 20 Aug 2024 11:56:30 -0700
Subject: [PATCH 34/99] FrameFile: Improve error messages. (#16912)

* FrameFile: Improve error messages.

1) Include frame file path in error messages.

2) Adhere better to style (no space before brackets).

* Fix test.
---
 .../java/org/apache/druid/frame/file/FrameFile.java  | 12 ++++++------
 .../apache/druid/frame/file/FrameFileWriterTest.java |  4 +++-
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java b/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java
index a6fbda9829d..7d21a54d852 100644
--- a/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java
+++ b/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java
@@ -173,7 +173,7 @@ public class FrameFile implements Closeable
     final EnumSet<Flag> flagSet = flags.length == 0 ? EnumSet.noneOf(Flag.class) : EnumSet.copyOf(Arrays.asList(flags));
 
     if (!file.exists()) {
-      throw new FileNotFoundException(StringUtils.format("File [%s] not found", file));
+      throw new FileNotFoundException(StringUtils.format("File[%s] not found", file));
     }
 
     // Closer for mmap that is shared across all references: either footer only (if file size is larger
@@ -186,7 +186,7 @@ public class FrameFile implements Closeable
       // Verify minimum file length.
       if (fileLength <
           FrameFileWriter.MAGIC.length + FrameFileWriter.TRAILER_LENGTH + Byte.BYTES /* MARKER_NO_MORE_FRAMES */) {
-        throw new IOE("File [%s] is too short (size = [%,d])", file, fileLength);
+        throw new IOE("File[%s] is too short (size[%,d])", file, fileLength);
       }
 
       // Verify magic.
@@ -195,7 +195,7 @@ public class FrameFile implements Closeable
       randomAccessFile.readFully(buf, 0, FrameFileWriter.MAGIC.length);
 
       if (!bufMemory.equalTo(0, Memory.wrap(FrameFileWriter.MAGIC), 0, FrameFileWriter.MAGIC.length)) {
-        throw new IOE("File [%s] is not a frame file", file);
+        throw new IOE("File[%s] is not a frame file", file);
       }
 
       // Read number of frames and partitions.
@@ -204,9 +204,9 @@ public class FrameFile implements Closeable
 
       final int footerLength = bufMemory.getInt(Integer.BYTES * 2L);
       if (footerLength < 0) {
-        throw new ISE("Negative-size footer. Corrupt or truncated file?");
+        throw new ISE("Negative-size footer. Corrupt or truncated file[%s]?", file);
       } else if (footerLength > fileLength) {
-        throw new ISE("Oversize footer. Corrupt or truncated file?");
+        throw new ISE("Oversize footer. Corrupt or truncated file[%s]?", file);
       }
 
       final Memory wholeFileMemory;
@@ -243,7 +243,7 @@ public class FrameFile implements Closeable
       if (flagSet.contains(Flag.DELETE_ON_CLOSE)) {
         fileCloser.register(() -> {
           if (!file.delete()) {
-            log.warn("Could not delete frame file [%s]", file);
+            log.warn("Could not delete frame file[%s]", file);
           }
           if (byteTracker != null) {
             // Only release the bytes taken by frames, we don't track the header and footer as of now.
diff --git a/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java b/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java
index 5f0e019712b..20d685f2227 100644
--- a/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java
@@ -75,7 +75,9 @@ public class FrameFileWriterTest extends InitializedNullHandlingTest
 
     MatcherAssert.assertThat(
         e,
-        ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("Corrupt or truncated file?"))
+        ThrowableMessageMatcher.hasMessage(
+            CoreMatchers.containsString("Negative-size footer. Corrupt or truncated file[")
+        )
     );
   }
 }

From 0ce1b6b22fbb1d6a9ab257b9a8ad4cd8637da933 Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Wed, 21 Aug 2024 10:06:04 +0530
Subject: [PATCH 35/99] MSQ window function: Take segment granularity into
 consideration to fix NPE issues with ingestion (#16854)

This PR changes the logic for window functions to use the resultShuffleSpecFactory for the last window stage.
---
 .../WindowOperatorQueryFrameProcessor.java    | 21 +++++-
 .../msq/querykit/WindowOperatorQueryKit.java  | 67 ++++++++++++++-----
 .../apache/druid/msq/exec/MSQWindowTest.java  | 64 ++++++++++++++----
 .../drill/window/queries/lag_func/lag_Fn_18.q |  7 +-
 .../window/queries/lead_func/lead_Fn_18.q     |  7 +-
 5 files changed, 132 insertions(+), 34 deletions(-)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
index 5fbfd3119d0..6b28c0263a8 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
@@ -31,6 +31,7 @@ import org.apache.druid.frame.processor.FrameProcessors;
 import org.apache.druid.frame.processor.FrameRowTooLargeException;
 import org.apache.druid.frame.processor.ReturnOrAwait;
 import org.apache.druid.frame.read.FrameReader;
+import org.apache.druid.frame.util.SettableLongVirtualColumn;
 import org.apache.druid.frame.write.FrameWriter;
 import org.apache.druid.frame.write.FrameWriterFactory;
 import org.apache.druid.java.util.common.Unit;
@@ -51,6 +52,8 @@ import org.apache.druid.query.rowsandcols.semantic.ColumnSelectorFactoryMaker;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.VirtualColumn;
+import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.NullableTypeStrategy;
 import org.apache.druid.segment.column.RowSignature;
 
@@ -85,6 +88,9 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
   private ResultRow outputRow = null;
   private FrameWriter frameWriter = null;
 
+  private final VirtualColumns frameWriterVirtualColumns;
+  private final SettableLongVirtualColumn partitionBoostVirtualColumn;
+
   // List of type strategies to compare the partition columns across rows.
   // Type strategies are pushed in the same order as column types in frameReader.signature()
   private final NullableTypeStrategy[] typeStrategies;
@@ -119,6 +125,16 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
     for (int i = 0; i < frameReader.signature().size(); i++) {
       typeStrategies[i] = frameReader.signature().getColumnType(i).get().getNullableStrategy();
     }
+
+    // Get virtual columns to be added to the frame writer.
+    this.partitionBoostVirtualColumn = new SettableLongVirtualColumn(QueryKitUtils.PARTITION_BOOST_COLUMN);
+    final List<VirtualColumn> frameWriterVirtualColumns = new ArrayList<>();
+    final VirtualColumn segmentGranularityVirtualColumn =
+        QueryKitUtils.makeSegmentGranularityVirtualColumn(jsonMapper, query);
+    if (segmentGranularityVirtualColumn != null) {
+      frameWriterVirtualColumns.add(segmentGranularityVirtualColumn);
+    }
+    this.frameWriterVirtualColumns = VirtualColumns.create(frameWriterVirtualColumns);
   }
 
   @Override
@@ -404,7 +420,9 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
     if (frameWriter == null) {
       final ColumnSelectorFactoryMaker csfm = ColumnSelectorFactoryMaker.fromRAC(rac);
       final ColumnSelectorFactory frameWriterColumnSelectorFactory = csfm.make(rowId);
-      frameWriter = frameWriterFactory.newFrameWriter(frameWriterColumnSelectorFactory);
+      final ColumnSelectorFactory frameWriterColumnSelectorFactoryWithVirtualColumns =
+          frameWriterVirtualColumns.wrap(frameWriterColumnSelectorFactory);
+      frameWriter = frameWriterFactory.newFrameWriter(frameWriterColumnSelectorFactoryWithVirtualColumns);
       currentAllocatorCapacity = frameWriterFactory.allocatorCapacity();
     }
   }
@@ -422,6 +440,7 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
       final boolean didAddToFrame = frameWriter.addSelection();
       if (didAddToFrame) {
         rowId.incrementAndGet();
+        partitionBoostVirtualColumn.setValue(partitionBoostVirtualColumn.getValue() + 1);
       } else if (frameWriter.getNumRows() == 0) {
         throw new FrameRowTooLargeException(currentAllocatorCapacity);
       } else {
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryKit.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryKit.java
index a814640f704..b3686359d2a 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryKit.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryKit.java
@@ -24,6 +24,7 @@ import org.apache.druid.frame.key.ClusterBy;
 import org.apache.druid.frame.key.KeyColumn;
 import org.apache.druid.frame.key.KeyOrder;
 import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.msq.exec.Limits;
 import org.apache.druid.msq.input.stage.StageInputSpec;
@@ -105,8 +106,14 @@ public class WindowOperatorQueryKit implements QueryKit<WindowOperatorQuery>
 
     final int firstStageNumber = Math.max(minStageNumber, queryDefBuilder.getNextStageNumber());
     final WindowOperatorQuery queryToRun = (WindowOperatorQuery) originalQuery.withDataSource(dataSourcePlan.getNewDataSource());
-    final int maxRowsMaterialized;
 
+    // Get segment granularity from query context, and create ShuffleSpec and RowSignature to be used for the final window stage.
+    final Granularity segmentGranularity = QueryKitUtils.getSegmentGranularityFromContext(jsonMapper, queryToRun.getContext());
+    final ClusterBy finalWindowClusterBy = computeClusterByForFinalWindowStage(segmentGranularity);
+    final ShuffleSpec finalWindowStageShuffleSpec = resultShuffleSpecFactory.build(finalWindowClusterBy, false);
+    final RowSignature finalWindowStageRowSignature = computeSignatureForFinalWindowStage(rowSignature, finalWindowClusterBy, segmentGranularity);
+
+    final int maxRowsMaterialized;
     if (originalQuery.context() != null && originalQuery.context().containsKey(MultiStageQueryContext.MAX_ROWS_MATERIALIZED_IN_WINDOW)) {
       maxRowsMaterialized = (int) originalQuery.context().get(MultiStageQueryContext.MAX_ROWS_MATERIALIZED_IN_WINDOW);
     } else {
@@ -122,13 +129,13 @@ public class WindowOperatorQueryKit implements QueryKit<WindowOperatorQuery>
       queryDefBuilder.add(
           StageDefinition.builder(firstStageNumber)
                          .inputs(new StageInputSpec(firstStageNumber - 1))
-                         .signature(rowSignature)
+                         .signature(finalWindowStageRowSignature)
                          .maxWorkerCount(maxWorkerCount)
-                         .shuffleSpec(null)
+                         .shuffleSpec(finalWindowStageShuffleSpec)
                          .processorFactory(new WindowOperatorQueryFrameProcessorFactory(
                              queryToRun,
                              queryToRun.getOperators(),
-                             rowSignature,
+                             finalWindowStageRowSignature,
                              maxRowsMaterialized,
                              Collections.emptyList()
                          ))
@@ -178,23 +185,22 @@ public class WindowOperatorQueryKit implements QueryKit<WindowOperatorQuery>
           }
         }
 
-        // find the shuffle spec of the next stage
-        // if it is the last stage set the next shuffle spec to single partition
-        if (i + 1 == operatorList.size()) {
-          nextShuffleSpec = MixShuffleSpec.instance();
-        } else {
-          nextShuffleSpec = findShuffleSpecForNextWindow(operatorList.get(i + 1), maxWorkerCount);
-        }
-
         final RowSignature intermediateSignature = bob.build();
         final RowSignature stageRowSignature;
-        if (nextShuffleSpec == null) {
-          stageRowSignature = intermediateSignature;
+
+        if (i + 1 == operatorList.size()) {
+          stageRowSignature = finalWindowStageRowSignature;
+          nextShuffleSpec = finalWindowStageShuffleSpec;
         } else {
-          stageRowSignature = QueryKitUtils.sortableSignature(
-              intermediateSignature,
-              nextShuffleSpec.clusterBy().getColumns()
-          );
+          nextShuffleSpec = findShuffleSpecForNextWindow(operatorList.get(i + 1), maxWorkerCount);
+          if (nextShuffleSpec == null) {
+            stageRowSignature = intermediateSignature;
+          } else {
+            stageRowSignature = QueryKitUtils.sortableSignature(
+                intermediateSignature,
+                nextShuffleSpec.clusterBy().getColumns()
+            );
+          }
         }
 
         log.info("Using row signature [%s] for window stage.", stageRowSignature);
@@ -346,4 +352,29 @@ public class WindowOperatorQueryKit implements QueryKit<WindowOperatorQuery>
     }
     return queryDefBuilder;
   }
+
+  /**
+   * Computes the ClusterBy for the final window stage. We don't have to take the CLUSTERED BY columns into account,
+   * as they are handled as {@link org.apache.druid.query.scan.ScanQuery#orderBys}.
+   */
+  private static ClusterBy computeClusterByForFinalWindowStage(Granularity segmentGranularity)
+  {
+    final List<KeyColumn> clusterByColumns = Collections.singletonList(new KeyColumn(QueryKitUtils.PARTITION_BOOST_COLUMN, KeyOrder.ASCENDING));
+    return QueryKitUtils.clusterByWithSegmentGranularity(new ClusterBy(clusterByColumns, 0), segmentGranularity);
+  }
+
+  /**
+   * Computes the signature for the final window stage. The finalWindowClusterBy will always have the
+   * partition boost column as computed in {@link #computeClusterByForFinalWindowStage(Granularity)}.
+   */
+  private static RowSignature computeSignatureForFinalWindowStage(RowSignature rowSignature, ClusterBy finalWindowClusterBy, Granularity segmentGranularity)
+  {
+    final RowSignature.Builder finalWindowStageRowSignatureBuilder = RowSignature.builder()
+                                                                                 .addAll(rowSignature)
+                                                                                 .add(QueryKitUtils.PARTITION_BOOST_COLUMN, ColumnType.LONG);
+    return QueryKitUtils.sortableSignature(
+        QueryKitUtils.signatureWithSegmentGranularity(finalWindowStageRowSignatureBuilder.build(), segmentGranularity),
+        finalWindowClusterBy.getColumns()
+    );
+  }
 }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java
index ee7121b3678..3a1e3d95f80 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java
@@ -1272,20 +1272,20 @@ public class MSQWindowTest extends MSQTestBase
         .setExpectedResultRows(
             NullHandling.replaceWithDefault() ?
             ImmutableList.of(
-                new Object[]{"", 11.0},
+                new Object[]{"a", 5.0},
                 new Object[]{"", 11.0},
                 new Object[]{"", 11.0},
                 new Object[]{"a", 5.0},
-                new Object[]{"a", 5.0},
-                new Object[]{"abc", 5.0}
+                new Object[]{"abc", 5.0},
+                new Object[]{"", 11.0}
             ) :
             ImmutableList.of(
-                new Object[]{null, 8.0},
+                new Object[]{"a", 5.0},
                 new Object[]{null, 8.0},
                 new Object[]{"", 3.0},
                 new Object[]{"a", 5.0},
-                new Object[]{"a", 5.0},
-                new Object[]{"abc", 5.0}
+                new Object[]{"abc", 5.0},
+                new Object[]{null, 8.0}
             ))
         .setQueryContext(context)
         .verifyResults();
@@ -1935,11 +1935,11 @@ public class MSQWindowTest extends MSQTestBase
                                    .build())
         .setExpectedRowSignature(rowSignature)
         .setExpectedResultRows(ImmutableList.of(
-            new Object[]{"Al Ain", 8L, 6334L},
-            new Object[]{"Dubai", 3L, 6334L},
-            new Object[]{"Dubai", 6323L, 6334L},
-            new Object[]{"Tirana", 26L, 26L},
-            new Object[]{"Benguela", 0L, 0L}
+            new Object[]{"Auburn", 0L, 1698L},
+            new Object[]{"Mexico City", 0L, 6136L},
+            new Object[]{"Seoul", 663L, 5582L},
+            new Object[]{"Tokyo", 0L, 12615L},
+            new Object[]{"Santiago", 161L, 401L}
         ))
         .setQueryContext(context)
         .verifyResults();
@@ -2266,13 +2266,13 @@ public class MSQWindowTest extends MSQTestBase
             2, 0, "input0"
         )
         .setExpectedCountersForStageWorkerChannel(
-            CounterSnapshotMatcher.with().rows(13).bytes(1158).frames(1),
+            CounterSnapshotMatcher.with().rows(13).bytes(1379).frames(1),
             2, 0, "output"
         )
 
         // Stage 3, Worker 0
         .setExpectedCountersForStageWorkerChannel(
-            CounterSnapshotMatcher.with().rows(13).bytes(1158).frames(1),
+            CounterSnapshotMatcher.with().rows(13).bytes(1327).frames(1),
             3, 0, "input0"
         )
         .setExpectedCountersForStageWorkerChannel(
@@ -2285,4 +2285,42 @@ public class MSQWindowTest extends MSQTestBase
         )
         .verifyResults();
   }
+
+  @MethodSource("data")
+  @ParameterizedTest(name = "{index}:with context {0}")
+  public void testReplaceWithPartitionedByDayOnWikipedia(String contextName, Map<String, Object> context)
+  {
+    RowSignature rowSignature = RowSignature.builder()
+                                            .add("__time", ColumnType.LONG)
+                                            .add("cityName", ColumnType.STRING)
+                                            .add("added", ColumnType.LONG)
+                                            .add("cc", ColumnType.LONG)
+                                            .build();
+
+    testIngestQuery().setSql(" REPLACE INTO foo1 OVERWRITE ALL\n"
+                             + "select __time, cityName, added, SUM(added) OVER () cc from wikipedia \n"
+                             + "where cityName IN ('Ahmedabad', 'Albuquerque')\n"
+                             + "GROUP BY __time, cityName, added\n"
+                             + "PARTITIONED BY DAY")
+                     .setExpectedDataSource("foo1")
+                     .setExpectedRowSignature(rowSignature)
+                     .setQueryContext(context)
+                     .setExpectedDestinationIntervals(Intervals.ONLY_ETERNITY)
+                     .setExpectedResultRows(
+                         ImmutableList.of(
+                             new Object[]{1442055085114L, "Ahmedabad", 0L, 140L},
+                             new Object[]{1442061929238L, "Ahmedabad", 0L, 140L},
+                             new Object[]{1442069353218L, "Albuquerque", 129L, 140L},
+                             new Object[]{1442069411614L, "Albuquerque", 9L, 140L},
+                             new Object[]{1442097803851L, "Albuquerque", 2L, 140L}
+                         )
+                     )
+                     .setExpectedSegments(ImmutableSet.of(SegmentId.of(
+                         "foo1",
+                         Intervals.of("2015-09-12/2015-09-13"),
+                         "test",
+                         0
+                     )))
+                     .verifyResults();
+  }
 }
diff --git a/sql/src/test/resources/drill/window/queries/lag_func/lag_Fn_18.q b/sql/src/test/resources/drill/window/queries/lag_func/lag_Fn_18.q
index e24ed5379df..a1bb3a29536 100644
--- a/sql/src/test/resources/drill/window/queries/lag_func/lag_Fn_18.q
+++ b/sql/src/test/resources/drill/window/queries/lag_func/lag_Fn_18.q
@@ -1 +1,6 @@
-SELECT col2 , col8 , LAG(col8 ) OVER ( PARTITION BY col2 ORDER BY col2 , col8 nulls FIRST ) LAG_col8 FROM "fewRowsAllData.parquet" FETCH FIRST 15 ROWS ONLY
+SELECT
+col2, col8,
+LAG(col8) OVER (PARTITION BY col2 ORDER BY col2, col8 nulls FIRST) LAG_col8
+FROM "fewRowsAllData.parquet"
+ORDER BY col2, col8
+FETCH FIRST 15 ROWS ONLY
diff --git a/sql/src/test/resources/drill/window/queries/lead_func/lead_Fn_18.q b/sql/src/test/resources/drill/window/queries/lead_func/lead_Fn_18.q
index e8a15695382..5249f290fae 100644
--- a/sql/src/test/resources/drill/window/queries/lead_func/lead_Fn_18.q
+++ b/sql/src/test/resources/drill/window/queries/lead_func/lead_Fn_18.q
@@ -1 +1,6 @@
-SELECT col2 , col8 , LEAD(col8 ) OVER ( PARTITION BY col2 ORDER BY col8 nulls FIRST ) LEAD_col8 FROM "fewRowsAllData.parquet" limit 10
+SELECT
+col2, col8,
+LEAD(col8) OVER (PARTITION BY col2 ORDER BY col8 nulls FIRST) LEAD_col8
+FROM "fewRowsAllData.parquet"
+ORDER BY col2, col8
+limit 10

From f6adacf5d604289bd5383ea522aed224c697c6bd Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Tue, 20 Aug 2024 23:10:29 -0700
Subject: [PATCH 36/99] SuperSorter: Store readOnly output channels. (#16928)

Without the call to readOnly, each output channel retains a 1 MB allocator,
leading to excessive memory use. Fixes regression from #16775.
---
 .../main/java/org/apache/druid/frame/processor/SuperSorter.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java b/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
index 00be9a3166a..5cb9c0830e4 100644
--- a/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
+++ b/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
@@ -670,7 +670,7 @@ public class SuperSorter
         final OutputChannel outputChannel = outputChannelFactory.openChannel(intRank);
         writableChannel = outputChannel.getWritableChannel();
         frameAllocatorFactory = new SingleMemoryAllocatorFactory(outputChannel.getFrameMemoryAllocator());
-        outputChannels.set(intRank, outputChannel);
+        outputChannels.set(intRank, outputChannel.readOnly());
 
         if (totalMergingLevels == 1) {
           // Reading from the inputBuffer. (i.e. "direct mode"; see class-level javadoc for more details.)

From 97f9502ad2d43f4682dcd8b12d4468b1b7a23640 Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Wed, 21 Aug 2024 15:47:50 +0530
Subject: [PATCH 37/99] Enable MSQ WF drill tests which were previously
 disabled (#16935)

---
 .../druid/sql/calcite/DrillWindowQueryTest.java | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
index 9bf56d97d38..c1504e8b58e 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
@@ -4264,7 +4264,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_aggregates_winFnQry_83()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6155,7 +6154,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_aggregates_winFnQry_84()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6163,7 +6161,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_aggregates_winFnQry_85()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6549,7 +6546,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_avg_mulwds()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6557,7 +6553,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_count_mulwds()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6565,7 +6560,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_fval_mulwds()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6573,7 +6567,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_lval_mulwds()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6581,7 +6574,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_mulwind_08()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6589,7 +6581,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_mulwind_09()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -6597,7 +6588,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_sum_mulwds()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -7370,7 +7360,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_rnkNoFrm01()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -7378,7 +7367,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_rnkNoFrm02()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -7386,7 +7374,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_rnkNoFrm03()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -7394,7 +7381,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_rnkNoFrm04()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -7402,7 +7388,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_rnkNoFrm05()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -7410,7 +7395,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_frameclause_multipl_wnwds_rnkNoFrm06()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 
@@ -7523,7 +7507,6 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   @Test
   public void test_nestedAggs_multiWin_6()
   {
-    msqIncompatible();
     windowQueryTest();
   }
 

From 090023609b1468c4142a38ede5f34ac15269ff7f Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Wed, 21 Aug 2024 13:45:01 -0700
Subject: [PATCH 38/99] Loosen case in FrameFileWriterTest. (#16938)

The specific error on a truncated file can vary based on how the final
frame of the truncated file is written. This patch loosens the check so
it passes regardless of how the truncated file is written.
---
 .../java/org/apache/druid/frame/file/FrameFileWriterTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java b/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java
index 20d685f2227..972371d0fde 100644
--- a/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/file/FrameFileWriterTest.java
@@ -76,7 +76,7 @@ public class FrameFileWriterTest extends InitializedNullHandlingTest
     MatcherAssert.assertThat(
         e,
         ThrowableMessageMatcher.hasMessage(
-            CoreMatchers.containsString("Negative-size footer. Corrupt or truncated file[")
+            CoreMatchers.containsString("Corrupt or truncated file[")
         )
     );
   }

From 338da67bc6e1a0034316131370eafdb81219d755 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Wed, 21 Aug 2024 15:07:24 -0700
Subject: [PATCH 39/99] Add type coercion and null check to left, right, repeat
 exprs. (#16480)

* Add type coercion and null check to left, right, repeat exprs.

These exprs shouldn't validate types; they should coerce types. Coercion
is typical behavior for functions because it enables schema evolution.

The functions are also modified to check isNumericNull on the right-hand
argument. This was missing previously, which would erroneously cause
nulls to be treated as zeroes.

* Fix tests.
---
 .../org/apache/druid/math/expr/Function.java  | 45 ++++++-------
 .../apache/druid/math/expr/FunctionTest.java  | 53 ++++++++++++++-
 .../calcite/expression/ExpressionsTest.java   | 65 +------------------
 3 files changed, 76 insertions(+), 87 deletions(-)

diff --git a/processing/src/main/java/org/apache/druid/math/expr/Function.java b/processing/src/main/java/org/apache/druid/math/expr/Function.java
index d62d632f40b..a7fc7f8c186 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/Function.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/Function.java
@@ -368,20 +368,24 @@ public interface Function extends NamedFunction
 
   /**
    * Base class for a 2 variable input {@link Function} whose first argument is a {@link ExprType#STRING} and second
-   * argument is {@link ExprType#LONG}
+   * argument is {@link ExprType#LONG}. These functions return null if either argument is null.
    */
   abstract class StringLongFunction extends BivariateFunction
   {
     @Override
     protected final ExprEval eval(ExprEval x, ExprEval y)
     {
-      if (!x.type().is(ExprType.STRING) || !y.type().is(ExprType.LONG)) {
-        throw validationFailed("needs a STRING as first argument and a LONG as second argument");
+      final String xString = x.asString();
+      if (xString == null) {
+        return ExprEval.of(null);
       }
-      return eval(x.asString(), y.asInt());
+      if (y.isNumericNull()) {
+        return ExprEval.of(null);
+      }
+      return eval(xString, y.asLong());
     }
 
-    protected abstract ExprEval eval(@Nullable String x, int y);
+    protected abstract ExprEval eval(String x, long y);
   }
 
   /**
@@ -2824,16 +2828,14 @@ public interface Function extends NamedFunction
     }
 
     @Override
-    protected ExprEval eval(@Nullable String x, int y)
+    protected ExprEval eval(String x, long y)
     {
-      if (y < 0) {
+      int yInt = (int) y;
+      if (y < 0 || yInt != y) {
         throw validationFailed("needs a positive integer as the second argument");
       }
-      if (x == null) {
-        return ExprEval.of(null);
-      }
       int len = x.length();
-      return ExprEval.of(y < len ? x.substring(len - y) : x);
+      return ExprEval.of(y < len ? x.substring(len - yInt) : x);
     }
   }
 
@@ -2853,15 +2855,13 @@ public interface Function extends NamedFunction
     }
 
     @Override
-    protected ExprEval eval(@Nullable String x, int y)
+    protected ExprEval eval(String x, long y)
     {
-      if (y < 0) {
-        throw validationFailed("needs a postive integer as second argument");
+      int yInt = (int) y;
+      if (yInt < 0 || yInt != y) {
+        throw validationFailed("needs a positive integer as the second argument");
       }
-      if (x == null) {
-        return ExprEval.of(null);
-      }
-      return ExprEval.of(y < x.length() ? x.substring(0, y) : x);
+      return ExprEval.of(y < x.length() ? x.substring(0, yInt) : x);
     }
   }
 
@@ -3005,12 +3005,13 @@ public interface Function extends NamedFunction
     }
 
     @Override
-    protected ExprEval eval(String x, int y)
+    protected ExprEval eval(String x, long y)
     {
-      if (x == null) {
-        return ExprEval.of(null);
+      int yInt = (int) y;
+      if (yInt != y) {
+        throw validationFailed("needs an integer as the second argument");
       }
-      return ExprEval.of(y < 1 ? NullHandling.defaultStringValue() : StringUtils.repeat(x, y));
+      return ExprEval.of(y < 1 ? NullHandling.defaultStringValue() : StringUtils.repeat(x, yInt));
     }
   }
 
diff --git a/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java
index 521c00f2f17..bc74227e5f4 100644
--- a/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java
+++ b/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java
@@ -40,7 +40,6 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import javax.annotation.Nullable;
-
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.nio.ByteBuffer;
@@ -962,13 +961,65 @@ public class FunctionTest extends InitializedNullHandlingTest
     assertExpr("bitwiseConvertDoubleToLongBits(null)", null);
   }
 
+  @Test
+  public void testLeft()
+  {
+    assertExpr("left('hello', 0)", NullHandling.sqlCompatible() ? "" : null);
+    assertExpr("left('hello', 2)", "he");
+    assertExpr("left('hello', '2')", "he");
+    assertExpr("left('hello', 'hello')", null);
+    assertExpr("left('hello', 10)", "hello");
+    assertExpr("left('hello', null)", null);
+    assertExpr("left(31337, 2)", "31");
+    assertExpr("left(null, 10)", null);
+    assertExpr("left(nonexistent, 10)", null);
+
+    Throwable t1 = Assert.assertThrows(
+        DruidException.class,
+        () -> assertExpr("left('foo', -2)", null)
+    );
+    Assert.assertEquals("Function[left] needs a positive integer as the second argument", t1.getMessage());
+  }
+
+  @Test
+  public void testRight()
+  {
+    assertExpr("right('hello', 0)", NullHandling.sqlCompatible() ? "" : null);
+    assertExpr("right('hello', 2)", "lo");
+    assertExpr("right('hello', '2')", "lo");
+    assertExpr("right('hello', 'hello')", null);
+    assertExpr("right('hello', 10)", "hello");
+    assertExpr("right('hello', null)", null);
+    assertExpr("right(31337, 2)", "37");
+    assertExpr("right(null, 10)", null);
+    assertExpr("right(nonexistent, 10)", null);
+
+    Throwable t1 = Assert.assertThrows(
+        DruidException.class,
+        () -> assertExpr("right('foo', -2)", null)
+    );
+    Assert.assertEquals("Function[right] needs a positive integer as the second argument", t1.getMessage());
+  }
+
   @Test
   public void testRepeat()
   {
+    assertExpr("repeat('hello', 0)", null);
     assertExpr("repeat('hello', 2)", "hellohello");
+    assertExpr("repeat('hello', '2')", "hellohello");
+    assertExpr("repeat('hello', 'hello')", null);
     assertExpr("repeat('hello', -1)", null);
+    assertExpr("repeat('hello', null)", null);
     assertExpr("repeat(null, 10)", null);
     assertExpr("repeat(nonexistent, 10)", null);
+    assertExpr("repeat(4, 3)", "444");
+    assertExpr("repeat(4.1, 3)", "4.14.14.1");
+
+    Throwable t = Assert.assertThrows(
+        DruidException.class,
+        () -> assertExpr("repeat('foo', 9999999999)", null)
+    );
+    Assert.assertEquals("Function[repeat] needs an integer as the second argument", t.getMessage());
   }
 
   @Test
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java
index c94eac39703..3b5b9c1de13 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java
@@ -2371,27 +2371,6 @@ public class ExpressionsTest extends CalciteTestBase
     Assert.assertEquals("Function[right] needs a positive integer as the second argument", t.getMessage());
   }
 
-  @Test
-  public void testAbnormalRightWithWrongType()
-  {
-    Throwable t = Assert.assertThrows(
-        DruidException.class,
-        () -> testHelper.testExpressionString(
-            new RightOperatorConversion().calciteOperator(),
-            ImmutableList.of(
-                testHelper.makeInputRef("s"),
-                testHelper.makeInputRef("s")
-            ),
-            makeExpression("right(\"s\",\"s\")"),
-            null
-        )
-    );
-    Assert.assertEquals(
-        "Function[right] needs a STRING as first argument and a LONG as second argument",
-        t.getMessage()
-    );
-  }
-
   @Test
   public void testLeft()
   {
@@ -2461,28 +2440,7 @@ public class ExpressionsTest extends CalciteTestBase
             null
         )
     );
-    Assert.assertEquals("Function[left] needs a postive integer as second argument", t.getMessage());
-  }
-
-  @Test
-  public void testAbnormalLeftWithWrongType()
-  {
-    Throwable t = Assert.assertThrows(
-        DruidException.class,
-        () -> testHelper.testExpressionString(
-            new LeftOperatorConversion().calciteOperator(),
-            ImmutableList.of(
-                testHelper.makeInputRef("s"),
-                testHelper.makeInputRef("s")
-            ),
-            makeExpression("left(\"s\",\"s\")"),
-            null
-        )
-    );
-    Assert.assertEquals(
-        "Function[left] needs a STRING as first argument and a LONG as second argument",
-        t.getMessage()
-    );
+    Assert.assertEquals("Function[left] needs a positive integer as the second argument", t.getMessage());
   }
 
   @Test
@@ -2519,27 +2477,6 @@ public class ExpressionsTest extends CalciteTestBase
     );
   }
 
-  @Test
-  public void testAbnormalRepeatWithWrongType()
-  {
-    Throwable t = Assert.assertThrows(
-        DruidException.class,
-        () -> testHelper.testExpressionString(
-            new RepeatOperatorConversion().calciteOperator(),
-            ImmutableList.of(
-                testHelper.makeInputRef("s"),
-                testHelper.makeInputRef("s")
-            ),
-            makeExpression("repeat(\"s\",\"s\")"),
-            null
-        )
-    );
-    Assert.assertEquals(
-        "Function[repeat] needs a STRING as first argument and a LONG as second argument",
-        t.getMessage()
-    );
-  }
-
   @Test
   public void testOperatorConversionsDruidUnaryLongFn()
   {

From 725695342c977a597ba7819fe2e20f856a98f89e Mon Sep 17 00:00:00 2001
From: Edgar Melendrez <evmelendrez@gmail.com>
Date: Wed, 21 Aug 2024 15:08:25 -0700
Subject: [PATCH 40/99] [Docs] Batch07: adding examples to string functions
 (#16862)

* Lower,Upper,Lpad,Rpad,Parse_long

* up to REGEXP_EXTRACT

* batch 07 ready for review

* updated definitions in scalar

* Apply suggestions from code review

Co-authored-by: Charles Smith <techdocsmith@gmail.com>

* rpad and lpad

* addressing comments

* minor fixes

* improving examples based on suggestions

* matched -> matches

* correcting typo

* Apply suggestions from code review

Co-authored-by: Charles Smith <techdocsmith@gmail.com>

---------

Co-authored-by: Charles Smith <techdocsmith@gmail.com>
---
 docs/querying/sql-functions.md | 256 +++++++++++++++++++++++++++++----
 docs/querying/sql-scalar.md    |   8 +-
 2 files changed, 230 insertions(+), 34 deletions(-)

diff --git a/docs/querying/sql-functions.md b/docs/querying/sql-functions.md
index 2a814337176..82f597b963a 100644
--- a/docs/querying/sql-functions.md
+++ b/docs/querying/sql-functions.md
@@ -1506,11 +1506,13 @@ SELECT
 FROM "taxi-trips"
 LIMIT 1
 ```
+
 Returns the following:
 
 | `max_temperature` | `natural_log_max_temp` | 
 | -- | -- | 
 | `76` | `4.330733340286331` | 
+
 </details>
 
 [Learn more](sql-scalar.md#numeric-functions)
@@ -1552,19 +1554,65 @@ Looks up the expression in a registered query-time lookup table.
 
 ## LOWER
 
-`LOWER(expr)`
-
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
-
 Returns the expression in lowercase.
 
+* **Syntax:** `LOWER(expr)`
+* **Function type:** Scalar, string
+
+<details><summary>Example</summary>
+
+The following example converts the `OriginCityName` column from the `flight-carriers` datasource to lowercase.
+
+```sql
+SELECT 
+  "OriginCityName" AS "origin_city",
+  LOWER("OriginCityName") AS "lowercase"
+FROM "flight-carriers"
+LIMIT 1
+```
+
+Returns the following:
+
+| `origin_city` | `lowercase` |
+| -- | -- |
+`San Juan, PR` | `san juan, pr` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
+
+
 ## LPAD
 
-`LPAD(<CHARACTER>, <INTEGER>, [<CHARACTER>])`
+Returns a string of size `length` from `expr`. When the length of `expr` is less than `length`, left pads `expr` with `chars`, which defaults to the space character. Truncates `expr` to `length` if `length` is shorter than the length of `expr`.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `LPAD(expr, length[, chars])`
+* **Function type:** Scalar, string
+
+<details><summary>Example</summary>
+
+The following example left pads the value of `OriginStateName` from the `flight-carriers` datasource to return a total of 11 characters.
+
+```sql
+SELECT 
+  "OriginStateName" AS "origin_state",
+  LPAD("OriginStateName", 11, '+') AS "add_left_padding"
+FROM "flight-carriers"
+LIMIT 3
+```
+
+Returns the following:
+
+| `origin_state` | `add_left_padding` |
+| -- | -- |
+| `Puerto Rico` | `Puerto Rico` |
+| `Massachusetts` | `Massachuset` |
+| `Florida` | `++++Florida` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
-Returns the leftmost number of characters from an expression, optionally padded with the given characters.
 
 ## LTRIM
 
@@ -1794,11 +1842,31 @@ Parses `expr` into a `COMPLEX<json>` object. This operator deserializes JSON val
 
 ## PARSE_LONG
 
-`PARSE_LONG(<CHARACTER>, [<INTEGER>])`
+Converts a string into a long(BIGINT) with the given radix, or into DECIMAL(base 10) if a radix is not provided.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:**`PARSE_LONG(string[, radix])`
+* **Function type:** Scalar, string
+
+<details><summary>Example</summary>
+
+The following example converts the string representation of the binary, radix 2, number `1100` into its long (BIGINT) equivalent.
+
+```sql
+SELECT 
+  '1100' AS "binary_as_string", 
+  PARSE_LONG('1110', 2) AS "bigint_value"
+```
+
+Returns the following:
+
+| `binary_as_string` | `bigint_value` |
+| -- | -- |
+| `1100` | `14` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
-Converts a string into a BIGINT with the given base or into a DECIMAL data type if the base is not specified.
 
 ## PERCENT_RANK
 
@@ -1810,11 +1878,33 @@ Returns the relative rank of the row calculated as a percentage according to the
 
 ## POSITION
 
-`POSITION(<CHARACTER> IN <CHARACTER> [FROM <INTEGER>])`
+Returns the one-based index position of a substring within an expression, optionally starting from a given one-based index. If `substring` is not found, returns 0.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax**: `POSITION(substring IN expr [FROM startingIndex])`
+* **Function type:** Scalar, string
 
-Returns the one-based index position of a substring within an expression, optionally starting from a given one-based index.
+<details><summary>Example</summary>
+
+The following example returns the one-based index of the substring `PR` in the `OriginCityName` column from the `flight-carriers` datasource starting from index 5.
+
+```sql
+SELECT 
+  "OriginCityName" AS "origin_city",
+  POSITION('PR' IN "OriginCityName" FROM 5) AS "index"
+FROM "flight-carriers"
+LIMIT 2
+```
+
+Returns the following:
+
+| `origin_city` | `index` |
+| -- | -- |
+| `San Juan, PR` | `11` |
+| `Boston, MA` | `0` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## POWER
 
@@ -1872,28 +1962,90 @@ Returns the rank with gaps for a row within a window. For example, if two rows t
 
 ## REGEXP_EXTRACT
 
-`REGEXP_EXTRACT(<CHARACTER>, <CHARACTER>, [<INTEGER>])`
+Apply regular expression `pattern` to `expr` and extract the Nth capture group. If `N` is unspecified or zero, returns the first substring that matches the pattern. Returns `null` if there is no matching pattern.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `REGEXP_EXTRACT(expr, pattern[, N])`
+* **Function type:** Scalar, string 
 
-Applies a regular expression to the string expression and returns the _n_th match.
+<details><summary>Example</summary>
+
+The following example uses regular expressions to find city names inside the `OriginCityName` column from the `flight-carriers` datasource by matching what comes before the comma.
+
+```sql
+SELECT 
+  "OriginCityName" AS "origin_city",
+  REGEXP_EXTRACT("OriginCityName", '([^,]+)', 0) AS "pattern_match"
+FROM "flight-carriers"
+LIMIT 1
+```
+
+Returns the following:
+
+| `origin_city` | `pattern_match` |
+| -- | -- |
+| `San Juan, PR` | `San Juan`|
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## REGEXP_LIKE
 
-`REGEXP_LIKE(<CHARACTER>, <CHARACTER>)`
+Returns `true` if the regular expression `pattern` finds a match in `expr`. Returns `false` otherwise.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `REGEXP_LIKE(expr, pattern)`
+* **Function type:** Scalar, string
 
-Returns true or false signifying whether the regular expression finds a match in the string expression.
+<details><summary>Example</summary>
+
+The following example returns `true` when the `OriginCityName` column from `flight-carriers` has a city name containing a space.
+
+```sql
+SELECT 
+  "OriginCityName" AS "origin_city",
+  REGEXP_LIKE("OriginCityName", '[A-Za-z]+\s[A-Za-z]+') AS "pattern_found"
+FROM "flight-carriers"
+LIMIT 2
+```
+
+Returns the following:
+
+| `origin_city` | `pattern_found` |
+| -- | -- |
+| `San Juan, PR` | `true` |
+| `Boston, MA` | `false` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## REGEXP_REPLACE
 
-`REGEXP_REPLACE(<CHARACTER>, <CHARACTER>, <CHARACTER>)`
+Replaces all occurrences of a regular expression in a string expression with a replacement string. Refer to capture groups in the replacement string using `$group` syntax. For example: `$1` or `$2`.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `REGEXP_REPLACE(expr, pattern, replacement)`
+* **Function type:** Scalar, string
+
+<details><summary>Example</summary>
+
+The following example matches three consecutive words, where each word is its own capture group, and replaces the matched words with the word in the second capture group punctuated with exclamation marks.
+
+```sql
+SELECT 
+  'foo bar baz' AS "original_string",
+  REGEXP_REPLACE('foo bar baz', '([A-Za-z]+) ([A-Za-z]+) ([A-Za-z]+)' , '$2!') AS "modified_string"
+```
+
+Returns the following:
+
+| `original_string` | `modified_string` |
+| -- | -- |
+| `foo bar baz` | `bar!` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
-Replaces all occurrences of a regular expression in a string expression with a replacement string. The replacement
-string may refer to capture groups using `$1`, `$2`, etc.
 
 ## REPEAT
 
@@ -2041,11 +2193,34 @@ Returns the number of the row within the window starting from 1.
 
 ## RPAD
 
-`RPAD(<CHARACTER>, <INTEGER>, [<CHARACTER>])`
+Returns a string of size `length` from `expr`. When the length of `expr` is less than `length`, right pads `expr` with `chars`, which defaults to the space character. Truncates `expr` to `length` if `length` is shorter than the length of `expr`.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `RPAD(expr, length[, chars])`
+* **Function type:** Scalar, string
 
-Returns the rightmost number of characters from an expression, optionally padded with the given characters.
+<details><summary>Example</summary>
+
+The following example right pads the value of `OriginStateName` from the `flight-carriers` datasource to return a total of 11 characters.
+
+```sql
+SELECT 
+  "OriginStateName" AS "origin_state",
+  RPAD("OriginStateName", 11, '+') AS "add_right_padding"
+FROM "flight-carriers"
+LIMIT 3
+```
+
+Returns the following:
+
+| `origin_state` | `add_right_padding` |
+| -- | -- |
+| `Puerto Rico` | `Puerto Rico` |
+| `Massachusetts` | `Massachuset` |
+| `Florida` | `Florida++++` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## RTRIM
 
@@ -2586,12 +2761,33 @@ For more information, see [UNNEST](./sql.md#unnest).
 
 ## UPPER
 
-`UPPER(expr)`
-
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
-
 Returns the expression in uppercase.
 
+* **Syntax:** `UPPER(expr)`
+* **Function type:** Scalar, string
+
+<details><summary>Example</summary>
+
+The following example converts the `OriginCityName` column from the `flight-carriers` datasource to uppercase.
+
+```sql
+SELECT 
+  "OriginCityName" AS "origin_city",
+  UPPER("OriginCityName") AS "uppercase"
+FROM "flight-carriers"
+LIMIT 1
+```
+
+Returns the following:
+
+| `origin_city` | `uppercase` |
+| -- | -- |
+`San Juan, PR` | `SAN JUAN, PR` |
+
+</details>
+
+[Learn more](sql-scalar.md#string-functions)
+
 ## VAR_POP
 
 `VAR_POP(expr)`
diff --git a/docs/querying/sql-scalar.md b/docs/querying/sql-scalar.md
index 662c9dc9ed6..f590d97d0f8 100644
--- a/docs/querying/sql-scalar.md
+++ b/docs/querying/sql-scalar.md
@@ -105,11 +105,11 @@ String functions accept strings and return a type appropriate to the function.
 |`LOOKUP(expr, lookupName[, replaceMissingValueWith])`|Looks up `expr` in an existing [query-time lookup table](lookups.md) that has the name `lookupName`. Returns the optional constant `replaceMissingValueWith` when `expr` is null or when the lookup does not contain a value for `expr`.<br /><br />You can query lookups directly using the [`lookup` schema](sql.md#from).|
 |`LOWER(expr)`|Returns `expr` in all lowercase.|
 |`UPPER(expr)`|Returns `expr` in all uppercase.|
-|`LPAD(expr, length[, chars])`|Returns a string of `length` from `expr` left-padded with `chars`. If `length` is shorter than the length of `expr`, the result is `expr` which is truncated to `length`. The result is null if either `expr` or `chars` is null. If `chars` is an empty string, no padding is added; however, `expr` may be trimmed if necessary.|
-|`RPAD(expr, length[, chars])`|Returns a string of `length` from `expr` right-padded with `chars`. If `length` is shorter than the length of `expr`, the result is `expr` which is truncated to `length`. The result is null if either `expr` or `chars` is null. If `chars` is an empty string, no padding is added; however, `expr` may be trimmed if necessary.|
-|`PARSE_LONG(string[, radix])`|Parses a string into a long (BIGINT) with the given radix or 10 (decimal) if a radix is not provided.|
+|`LPAD(expr, length[, chars])`|Returns a string of `length` from `expr`. If `expr` is shorter than `length`, left pads `expr` with `chars`, which defaults to space characters. If `expr` exceeds `length`, truncates `expr` to equal `length`.  If `chars` is an empty string, no padding is added. Returns `null` if either `expr` or `chars` is null.|
+|`RPAD(expr, length[, chars])`|Returns a string of `length` from `expr`. If `expr` is shorter than `length`, right pads `expr` with `chars`, which defaults to space characters. If `expr` exceeds `length`, truncates `expr` to equal `length`.  If `chars` is an empty string, no padding is added. Returns `null` if either `expr` or `chars` is null.|
+|`PARSE_LONG(string[, radix])`|Parses a string into a long (BIGINT) with the given radix, or 10 (decimal) if a radix is not provided.|
 |`POSITION(substring IN expr [FROM startingIndex])`|Returns the index of `substring` within `expr` with indexes starting from 1. The search begins at `startingIndex`. If `startingIndex` is not specified, the default is 1. If `substring` is not found, returns 0.|
-|`REGEXP_EXTRACT(expr, pattern[, index])`|Apply regular expression `pattern` to `expr` and extract a capture group or `NULL` if there is no match. If `index` is unspecified or zero, returns the first substring that matched the pattern. The pattern may match anywhere inside `expr`. To match the entire string, use the `^` and `$` markers at the start and end of your pattern. When `druid.generic.useDefaultValueForNull = true`, it is not possible to differentiate an empty-string match from a non-match (both return `NULL`).|
+|`REGEXP_EXTRACT(expr, pattern[, index])`|Apply regular expression `pattern` to `expr` and extract a capture group or `NULL` if there is no match. If `index` is unspecified or zero, returns the first substring that matches the pattern. The pattern may match anywhere inside `expr`. To match the entire string, use the `^` and `$` markers at the start and end of your pattern. When `druid.generic.useDefaultValueForNull = true`, it is not possible to differentiate an empty-string match from a non-match (both return `NULL`).|
 |`REGEXP_LIKE(expr, pattern)`|Returns whether `expr` matches regular expression `pattern`. The pattern may match anywhere inside `expr`; if you want to match the entire string instead, use the `^` and `$` markers at the start and end of your pattern. Similar to [`LIKE`](sql-operators.md#logical-operators), but uses regexps instead of LIKE patterns. Especially useful in WHERE clauses.|
 |`REGEXP_REPLACE(expr, pattern, replacement)`|Replaces all occurrences of regular expression `pattern` within `expr` with `replacement`. The replacement string may refer to capture groups using `$1`, `$2`, etc. The pattern may match anywhere inside `expr`; if you want to match the entire string instead, use the `^` and `$` markers at the start and end of your pattern.|
 |`REPLACE(expr, substring, replacement)`|Replaces instances of `substring` in `expr` with `replacement` and returns the result.|

From a83125e4a00c56563709e664a4e9685a834c7ec8 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Wed, 21 Aug 2024 23:13:46 -0700
Subject: [PATCH 41/99] Track IngestionState more accurately in realtime tasks.
 (#16934)

Previously, SeekableStreamIndexTaskRunner set ingestion state to
COMPLETED when it finished reading data from Kafka. This is incorrect.
After the changes in this patch, the transitions go:

1) The task stays in BUILD_SEGMENTS after it finishes reading from Kafka,
   while it is building its final set of segments to publish.

2) The task transitions to SEGMENT_AVAILABILITY_WAIT after publishing,
   while waiting for handoff.

3) The task transitions to COMPLETED immediately before exiting, when
   truly done.
---
 docs/ingestion/tasks.md                       |  1 +
 .../indexing/kafka/KafkaIndexTaskTest.java    | 34 +++++++++++++++++--
 .../kinesis/KinesisIndexTaskTest.java         |  9 +++++
 .../SeekableStreamIndexTaskRunner.java        |  3 +-
 4 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/docs/ingestion/tasks.md b/docs/ingestion/tasks.md
index 3291abb01a0..743a46cc854 100644
--- a/docs/ingestion/tasks.md
+++ b/docs/ingestion/tasks.md
@@ -239,6 +239,7 @@ The `ingestionState` shows what step of ingestion the task reached. Possible sta
 - `NOT_STARTED`: The task has not begun reading any rows
 - `DETERMINE_PARTITIONS`: The task is processing rows to determine partitioning
 - `BUILD_SEGMENTS`: The task is processing rows to construct segments
+- `SEGMENT_AVAILABILITY_WAIT`: The task has published its segments and is waiting for them to become available.
 - `COMPLETED`: The task has finished its work.
 
 Only batch tasks have the DETERMINE_PARTITIONS phase. Realtime tasks such as those created by the Kafka Indexing Service do not have a DETERMINE_PARTITIONS phase.
diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java
index 15b77be307d..f8c6b23aae9 100644
--- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java
+++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java
@@ -52,6 +52,7 @@ import org.apache.druid.data.input.kafka.KafkaRecordEntity;
 import org.apache.druid.data.input.kafka.KafkaTopicPartition;
 import org.apache.druid.data.input.kafkainput.KafkaInputFormat;
 import org.apache.druid.data.input.kafkainput.KafkaStringHeaderFormat;
+import org.apache.druid.indexer.IngestionState;
 import org.apache.druid.indexer.TaskState;
 import org.apache.druid.indexer.TaskStatus;
 import org.apache.druid.indexer.report.IngestionStatsAndErrors;
@@ -1617,6 +1618,10 @@ public class KafkaIndexTaskTest extends SeekableStreamIndexTaskTestBase
 
     IngestionStatsAndErrors reportData = getTaskReportData();
 
+    // Verify ingestion state and error message
+    Assert.assertEquals(IngestionState.COMPLETED, reportData.getIngestionState());
+    Assert.assertNull(reportData.getErrorMsg());
+
     Map<String, Object> expectedMetrics = ImmutableMap.of(
         RowIngestionMeters.BUILD_SEGMENTS,
         ImmutableMap.of(
@@ -1697,6 +1702,10 @@ public class KafkaIndexTaskTest extends SeekableStreamIndexTaskTestBase
 
     IngestionStatsAndErrors reportData = getTaskReportData();
 
+    // Verify ingestion state and error message
+    Assert.assertEquals(IngestionState.BUILD_SEGMENTS, reportData.getIngestionState());
+    Assert.assertNotNull(reportData.getErrorMsg());
+
     Map<String, Object> expectedMetrics = ImmutableMap.of(
         RowIngestionMeters.BUILD_SEGMENTS,
         ImmutableMap.of(
@@ -3057,9 +3066,13 @@ public class KafkaIndexTaskTest extends SeekableStreamIndexTaskTestBase
         newDataSchemaMetadata()
     );
 
-    // Verify unparseable data
     IngestionStatsAndErrors reportData = getTaskReportData();
 
+    // Verify ingestion state and error message
+    Assert.assertEquals(IngestionState.COMPLETED, reportData.getIngestionState());
+    Assert.assertNull(reportData.getErrorMsg());
+
+    // Verify unparseable data
     ParseExceptionReport parseExceptionReport =
         ParseExceptionReport.forPhase(reportData, RowIngestionMeters.BUILD_SEGMENTS);
 
@@ -3190,9 +3203,14 @@ public class KafkaIndexTaskTest extends SeekableStreamIndexTaskTestBase
     Assert.assertEquals(ImmutableList.of(), publishedDescriptors());
     Assert.assertNull(newDataSchemaMetadata());
 
+    // Verify ingestion state and error message
+    final IngestionStatsAndErrors reportData = getTaskReportData();
+    Assert.assertEquals(IngestionState.BUILD_SEGMENTS, reportData.getIngestionState());
+    Assert.assertNotNull(reportData.getErrorMsg());
+
     // Verify there is no unparseable data in the report since we've 0 saved parse exceptions
     ParseExceptionReport parseExceptionReport =
-        ParseExceptionReport.forPhase(getTaskReportData(), RowIngestionMeters.BUILD_SEGMENTS);
+        ParseExceptionReport.forPhase(reportData, RowIngestionMeters.BUILD_SEGMENTS);
 
     Assert.assertEquals(ImmutableList.of(), parseExceptionReport.getErrorMessages());
   }
@@ -3231,6 +3249,12 @@ public class KafkaIndexTaskTest extends SeekableStreamIndexTaskTestBase
 
     Assert.assertEquals(TaskState.SUCCESS, status.getStatusCode());
     IngestionStatsAndErrors reportData = getTaskReportData();
+
+    // Verify ingestion state and error message
+    Assert.assertEquals(IngestionState.COMPLETED, reportData.getIngestionState());
+    Assert.assertNull(reportData.getErrorMsg());
+
+    // Verify report metrics
     Assert.assertEquals(reportData.getRecordsProcessed().size(), 1);
     Assert.assertEquals(reportData.getRecordsProcessed().values().iterator().next(), (Long) 6L);
   }
@@ -3279,6 +3303,12 @@ public class KafkaIndexTaskTest extends SeekableStreamIndexTaskTestBase
 
     Assert.assertEquals(TaskState.SUCCESS, status.getStatusCode());
     IngestionStatsAndErrors reportData = getTaskReportData();
+
+    // Verify ingestion state and error message
+    Assert.assertEquals(IngestionState.COMPLETED, reportData.getIngestionState());
+    Assert.assertNull(reportData.getErrorMsg());
+
+    // Verify report metrics
     Assert.assertEquals(reportData.getRecordsProcessed().size(), 2);
     Assert.assertTrue(reportData.getRecordsProcessed().values().containsAll(ImmutableSet.of(6L, 2L)));
   }
diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java
index 80bded2031d..510eaa797e0 100644
--- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java
+++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java
@@ -43,6 +43,7 @@ import org.apache.druid.data.input.impl.FloatDimensionSchema;
 import org.apache.druid.data.input.impl.LongDimensionSchema;
 import org.apache.druid.data.input.impl.StringDimensionSchema;
 import org.apache.druid.data.input.kinesis.KinesisRecordEntity;
+import org.apache.druid.indexer.IngestionState;
 import org.apache.druid.indexer.TaskState;
 import org.apache.druid.indexer.TaskStatus;
 import org.apache.druid.indexer.report.IngestionStatsAndErrors;
@@ -1186,6 +1187,10 @@ public class KinesisIndexTaskTest extends SeekableStreamIndexTaskTestBase
 
     IngestionStatsAndErrors reportData = getTaskReportData();
 
+    // Verify ingestion state and error message
+    Assert.assertEquals(IngestionState.COMPLETED, reportData.getIngestionState());
+    Assert.assertNull(reportData.getErrorMsg());
+
     Map<String, Object> expectedMetrics = ImmutableMap.of(
         RowIngestionMeters.BUILD_SEGMENTS,
         ImmutableMap.of(
@@ -1272,6 +1277,10 @@ public class KinesisIndexTaskTest extends SeekableStreamIndexTaskTestBase
 
     IngestionStatsAndErrors reportData = getTaskReportData();
 
+    // Verify ingestion state and error message
+    Assert.assertEquals(IngestionState.BUILD_SEGMENTS, reportData.getIngestionState());
+    Assert.assertNotNull(reportData.getErrorMsg());
+
     Map<String, Object> expectedMetrics = ImmutableMap.of(
         RowIngestionMeters.BUILD_SEGMENTS,
         ImmutableMap.of(
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java
index d347fd81503..df8e220145b 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java
@@ -768,7 +768,6 @@ public abstract class SeekableStreamIndexTaskRunner<PartitionIdType, SequenceOff
             }
           }
         }
-        ingestionState = IngestionState.COMPLETED;
       }
       catch (Exception e) {
         // (1) catch all exceptions while reading from kafka
@@ -835,6 +834,7 @@ public abstract class SeekableStreamIndexTaskRunner<PartitionIdType, SequenceOff
       // failed to persist sequences. It might also return null if handoff failed, but was recoverable.
       // See publishAndRegisterHandoff() for details.
       List<SegmentsAndCommitMetadata> handedOffList = Collections.emptyList();
+      ingestionState = IngestionState.SEGMENT_AVAILABILITY_WAIT;
       if (tuningConfig.getHandoffConditionTimeout() == 0) {
         handedOffList = Futures.allAsList(handOffWaitList).get();
       } else {
@@ -928,6 +928,7 @@ public abstract class SeekableStreamIndexTaskRunner<PartitionIdType, SequenceOff
       }
     }
 
+    ingestionState = IngestionState.COMPLETED;
     toolbox.getTaskReportFileWriter().write(task.getId(), getTaskCompletionReports(null, handoffWaitMs));
     return TaskStatus.success(task.getId());
   }

From fda2d19b8823918106f10b01ba9af54c5fe5a4e8 Mon Sep 17 00:00:00 2001
From: Edgar Melendrez <evmelendrez@gmail.com>
Date: Thu, 22 Aug 2024 11:11:16 -0700
Subject: [PATCH 42/99] [Docs] Batch09: only `lookup` (#16878)

* [Docs] Batch09: only `lookup`

* slight changes

* Apply suggestions from code review

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* applying suggestiontions

* Apply suggestions from code review

Co-authored-by: Victoria Lim <vtlim@users.noreply.github.com>

* otherwise null -> otherwise returns null

* updating definition in sql-scalar.md

* Apply suggestions from code review

Co-authored-by: Charles Smith <techdocsmith@gmail.com>

* hoping to re-run web checks

* change replaceMissingValueWith -> defaultValue

* Update docs/querying/sql-scalar.md

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* acronym_to_name -> airportcode_to_name

* shortens `airportcode_to_name` to `code_to_name`

---------

Co-authored-by: Katya Macedo <38017980+ektravel@users.noreply.github.com>
Co-authored-by: Victoria Lim <vtlim@users.noreply.github.com>
Co-authored-by: Charles Smith <techdocsmith@gmail.com>
---
 docs/querying/sql-functions.md | 37 +++++++++++++++++++++++++++++++---
 docs/querying/sql-scalar.md    |  2 +-
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/docs/querying/sql-functions.md b/docs/querying/sql-functions.md
index 82f597b963a..b020d48bf59 100644
--- a/docs/querying/sql-functions.md
+++ b/docs/querying/sql-functions.md
@@ -1546,11 +1546,42 @@ Returns the following:
 
 ## LOOKUP
 
-`LOOKUP(<CHARACTER>, <CHARACTER>[, <CHARACTER>])`
+Searches for `expr` in a registered [query-time lookup table](lookups.md) named `lookupName` and returns the mapped value. If `expr` is null or not contained in the lookup, returns `defaultValue` if supplied, otherwise returns null.
 
-**Function type:** [Scalar, string](sql-scalar.md#string-functions)
+* **Syntax:** `LOOKUP(expr, lookupName[, defaultValue])`
+* **Function type:** Scalar, string
 
-Looks up the expression in a registered query-time lookup table.
+<details><summary>Example</summary>
+
+The following example uses a `map` type lookup table named `code_to_name`, which contains the following key-value pairs:
+
+```json
+{
+  "SJU": "Luis Munoz Marin International Airport",
+  "IAD": "Dulles International Airport"
+}
+```
+
+The example uses `code_to_name` to map the `Origin` column from the `flight-carriers` datasource to the corresponding full airport name. Returns `key not found` if no matching key exists in the lookup table.
+
+```sql
+SELECT 
+  "Origin" AS "origin_airport",
+  LOOKUP("Origin", 'code_to_name','key not found') AS "full_airport_name"
+FROM "flight-carriers"
+LIMIT 2
+```
+
+Returns the following:
+
+| `origin_airport` | `full_airport_name` | 
+| -- | -- |
+| `SJU` | `Luis Munoz Marin International Airport` |
+| `BOS` | `key not found` |
+
+</details> 
+
+[Learn more](sql-scalar.md#string-functions)
 
 ## LOWER
 
diff --git a/docs/querying/sql-scalar.md b/docs/querying/sql-scalar.md
index f590d97d0f8..ee94ca769da 100644
--- a/docs/querying/sql-scalar.md
+++ b/docs/querying/sql-scalar.md
@@ -102,7 +102,7 @@ String functions accept strings and return a type appropriate to the function.
 |`CHAR_LENGTH(expr)`|Alias for `LENGTH`.|
 |`CHARACTER_LENGTH(expr)`|Alias for `LENGTH`.|
 |`STRLEN(expr)`|Alias for `LENGTH`.|
-|`LOOKUP(expr, lookupName[, replaceMissingValueWith])`|Looks up `expr` in an existing [query-time lookup table](lookups.md) that has the name `lookupName`. Returns the optional constant `replaceMissingValueWith` when `expr` is null or when the lookup does not contain a value for `expr`.<br /><br />You can query lookups directly using the [`lookup` schema](sql.md#from).|
+|`LOOKUP(expr, lookupName[, replaceMissingValueWith])`|Searches for `expr` in a registered [query-time lookup table](lookups.md) named `lookupName` and returns the mapped value. If `expr` is null or not contained in the lookup, returns `replaceMissingValueWith` if supplied, otherwise returns null.<br /><br />You can query lookups directly using the [`lookup` schema](sql.md#from).|
 |`LOWER(expr)`|Returns `expr` in all lowercase.|
 |`UPPER(expr)`|Returns `expr` in all uppercase.|
 |`LPAD(expr, length[, chars])`|Returns a string of `length` from `expr`. If `expr` is shorter than `length`, left pads `expr` with `chars`, which defaults to space characters. If `expr` exceeds `length`, truncates `expr` to equal `length`.  If `chars` is an empty string, no padding is added. Returns `null` if either `expr` or `chars` is null.|

From bce60b06740f627862f11188172ae0f03c55f3d4 Mon Sep 17 00:00:00 2001
From: Clint Wylie <cwylie@apache.org>
Date: Thu, 22 Aug 2024 13:34:28 -0700
Subject: [PATCH 43/99] fix flaky
 ParallelMergeCombiningSequenceTest.testTimeoutExceptionDueToStoppedReader
 when runner is very slow (#16932)

---
 .../common/guava/ParallelMergeCombiningSequenceTest.java     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/processing/src/test/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequenceTest.java b/processing/src/test/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequenceTest.java
index 293943dcca9..cbe84cb4eec 100644
--- a/processing/src/test/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequenceTest.java
+++ b/processing/src/test/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequenceTest.java
@@ -573,6 +573,7 @@ public class ParallelMergeCombiningSequenceTest
   public void testTimeoutExceptionDueToStoppedReader() throws InterruptedException
   {
     final int someSize = 150_000;
+    final int timeout = 5_000;
     List<TestingReporter> reporters = new ArrayList<>();
     for (int i = 0; i < 100; i++) {
       List<Sequence<IntPair>> input = new ArrayList<>();
@@ -588,7 +589,7 @@ public class ParallelMergeCombiningSequenceTest
           INT_PAIR_ORDERING,
           INT_PAIR_MERGE_FN,
           true,
-          1000,
+          timeout,
           0,
           TEST_POOL_SIZE,
           512,
@@ -605,7 +606,7 @@ public class ParallelMergeCombiningSequenceTest
     }
 
     // sleep until timeout
-    Thread.sleep(5000);
+    Thread.sleep(timeout);
     Assert.assertTrue(pool.awaitQuiescence(10, TimeUnit.SECONDS));
     Assert.assertTrue(pool.isQuiescent());
     Assert.assertFalse(pool.hasQueuedSubmissions());

From 2aef6ac685d4fc147ffd9a7bf21a7dc7a551b67e Mon Sep 17 00:00:00 2001
From: Clint Wylie <cwylie@apache.org>
Date: Thu, 22 Aug 2024 13:36:43 -0700
Subject: [PATCH 44/99] fix ipv4_parse function return type in SQL to be bigint
 instead of integer (#16942)

* fix ipv4_parse function return type in SQL to be bigint instead of integer

* fix default value mode
---
 .../IPv4AddressParseOperatorConversion.java   |  2 +-
 .../druid/sql/calcite/CalciteQueryTest.java   | 26 ++++++++++++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/IPv4AddressParseOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/IPv4AddressParseOperatorConversion.java
index 297f10c63ad..e4ad6e796b3 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/IPv4AddressParseOperatorConversion.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/IPv4AddressParseOperatorConversion.java
@@ -39,7 +39,7 @@ public class IPv4AddressParseOperatorConversion extends DirectOperatorConversion
               OperandTypes.family(SqlTypeFamily.STRING),
               OperandTypes.family(SqlTypeFamily.INTEGER)
           ))
-      .returnTypeNullable(SqlTypeName.INTEGER)
+      .returnTypeNullable(SqlTypeName.BIGINT)
       .functionCategory(SqlFunctionCategory.USER_DEFINED_FUNCTION)
       .build();
 
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 7566d7f7786..556b437e758 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -16128,7 +16128,31 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
                   .resultFormat(ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
                   .build()
         ),
-        ImmutableList.of(NullHandling.sqlCompatible() ? new Object[]{null} : new Object[]{0})
+        ImmutableList.of(NullHandling.sqlCompatible() ? new Object[]{null} : new Object[]{0L})
+    );
+  }
+
+  @Test
+  public void testIpv4ParseWithBigintOutput()
+  {
+    testQuery(
+        "select ipv4_parse('192.168.0.1') from (values(1)) as t(col)",
+        ImmutableList.of(
+            Druids.newScanQueryBuilder()
+                  .dataSource(InlineDataSource.fromIterable(
+                      ImmutableList.of(new Object[]{1L}),
+                      RowSignature.builder()
+                                  .add("col", ColumnType.LONG)
+                                  .build()
+                  ))
+                  .intervals(querySegmentSpec(Filtration.eternity()))
+                  .columns("v0")
+                  .virtualColumns(expressionVirtualColumn("v0", "3232235521", ColumnType.LONG))
+                  .context(QUERY_CONTEXT_DEFAULT)
+                  .resultFormat(ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
+                  .build()
+        ),
+        ImmutableList.of(new Object[]{3232235521L})
     );
   }
 

From 2f6722db6356d9ca0a5c25afcbf2022dffbcc02c Mon Sep 17 00:00:00 2001
From: Hugh Evans <101668034+hevansDev@users.noreply.github.com>
Date: Fri, 23 Aug 2024 02:09:09 +0100
Subject: [PATCH 45/99] Updated auth to use variables with default values
 (#16876)

* Updated auth to use variables with default values

* Update docs/api-reference/sql-ingestion-api.md

* Remove Python auth entirely as its not being used

---------

Co-authored-by: Benedict Jin <asdf2014@apache.org>
---
 docs/api-reference/sql-ingestion-api.md | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/api-reference/sql-ingestion-api.md b/docs/api-reference/sql-ingestion-api.md
index 275be9e63dc..a289aeefee7 100644
--- a/docs/api-reference/sql-ingestion-api.md
+++ b/docs/api-reference/sql-ingestion-api.md
@@ -157,7 +157,7 @@ headers = {
   'Content-Type': 'application/json'
 }
 
-response = requests.post(url, headers=headers, data=payload, auth=('USER', 'PASSWORD'))
+response = requests.post(url, headers=headers, data=payload)
 
 print(response.text)
 
@@ -254,8 +254,9 @@ url = "http://ROUTER_IP:ROUTER_PORT/druid/indexer/v1/task/query-3dc0c45d-34d7-4b
 payload={}
 headers = {}
 
-response = requests.get(url, headers=headers, data=payload, auth=('USER', 'PASSWORD'))
+response = requests.post(url, headers=headers, data=payload)
 
+print(response.text)
 print(response.text)
 ```
 
@@ -363,7 +364,10 @@ import requests
 url = "http://ROUTER_IP:ROUTER_PORT/druid/indexer/v1/task/query-3dc0c45d-34d7-4b15-86c9-cdb2d3ebfc4e/reports"
 
 headers = {}
-response = requests.get(url, headers=headers, auth=('USER', 'PASSWORD'))
+
+response = requests.post(url, headers=headers, data=payload)
+
+print(response.text)
 print(response.text)
 ```
 
@@ -826,11 +830,12 @@ import requests
 
 url = "http://ROUTER_IP:ROUTER_PORT/druid/indexer/v1/task/query-655efe33-781a-4c50-ae84-c2911b42d63c/shutdown"
 
-payload={}
+payload = {}
 headers = {}
 
-response = requests.post(url, headers=headers, data=payload, auth=('USER', 'PASSWORD'))
+response = requests.post(url, headers=headers, data=payload)
 
+print(response.text)
 print(response.text)
 ```
 

From c4981e34c47b54f3c57a38edbcb001a3b5e1bb6f Mon Sep 17 00:00:00 2001
From: Edgar Melendrez <evmelendrez@gmail.com>
Date: Thu, 22 Aug 2024 20:25:27 -0700
Subject: [PATCH 46/99] [docs] Batch10 date and time functions (#16900)

* just starting

* TIME_PARSE and TIME_FORMAT remaining

* fixing typo

* adding last two functions

* review sql-functions.md

* Apply suggestions from code review

Suggestions that were accepted as is

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* Update docs/querying/sql-functions.md

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* Update docs/querying/sql-functions.md

needed to confirm that it did indeed return as a number

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* reviewing remaining suggestions

* addressing review for time_format

* Apply suggestions from code review

Accepted as is

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

* addressing final suggestion

* time_zone -> timezone

* timezone fix

---------

Co-authored-by: Katya Macedo <38017980+ektravel@users.noreply.github.com>
---
 docs/querying/sql-functions.md | 242 +++++++++++++++++++++++++++++----
 docs/querying/sql-scalar.md    |  22 +--
 2 files changed, 225 insertions(+), 39 deletions(-)

diff --git a/docs/querying/sql-functions.md b/docs/querying/sql-functions.md
index b020d48bf59..abb17434a8e 100644
--- a/docs/querying/sql-functions.md
+++ b/docs/querying/sql-functions.md
@@ -826,28 +826,85 @@ Returns the cumulative distribution of the current row within the window calcula
 
 ## CURRENT_DATE
 
-`CURRENT_DATE`
+Returns the current date in UTC time, unless you specify a different timezone in the query context.
 
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
+* **Syntax:** `CURRENT_DATE`
+* **Function type:** Scalar, date and time
 
-Returns the current date in the connection's time zone.
+<details><summary>Example</summary>
+
+The following example returns the current date.
+
+```sql
+SELECT CURRENT_DATE AS "current_date"
+```
+
+Returns the following:
+
+| `current_date` | 
+| -- |
+| `2024-08-14T00:00:00.000Z `|
+
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
 
 ## CURRENT_TIMESTAMP
 
-`CURRENT_TIMESTAMP`
+Returns the current timestamp in UTC time, unless you specify a different timezone in the query context.
 
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
 
-Returns the current timestamp in the connection's time zone.
+* **Syntax:** `CURRENT_TIMESTAMP`
+* **Function type:** Scalar, date and time
+
+<details><summary>Example</summary>
+
+The following example returns the current timestamp.
+
+```sql
+SELECT CURRENT_TIMESTAMP AS "current_timestamp"
+```
+
+Returns the following:
+
+| `current_timestamp` |
+| -- |
+| `2024-08-14T21:30:13.793Z` |
+
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
 
 ## DATE_TRUNC
 
-`DATE_TRUNC(<CHARACTER>, <TIMESTAMP>)`
-
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
-
 Rounds down a timestamp by a given time unit.
 
+* **Syntax:** `DATE_TRUNC(unit, timestamp_expr)` 
+* **Function type:** Scalar, date and time
+
+<details><summary>Example</summary>
+
+The following example truncates a timestamp from the `__time` column from the `taxi-trips` datasource to the most recent `decade`.
+
+```sql
+SELECT 
+  "__time" AS "original_timestamp",
+  DATE_TRUNC('decade', "__time") AS "truncate_timestamp"
+FROM "taxi-trips"
+LIMIT 1
+```
+
+Returns the following:
+
+| `original_timestamp` | `truncate_time` |
+| -- | -- |
+| `2013-08-01T08:14:37.000Z` | `2010-01-01T00:00:00.000Z` |
+
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
+
+
 ## DECODE_BASE64_COMPLEX
 
 `DECODE_BASE64_COMPLEX(dataType, expr)`
@@ -2623,35 +2680,121 @@ Returns a union of Theta sketches.
 
 ## TIME_CEIL
 
-`TIME_CEIL(<TIMESTAMP>, <period>, [<origin>, [<timezone>]])`
+Rounds up a timestamp to a given ISO 8601 time period. You can specify `origin` to provide a reference timestamp from which to start rounding. If provided, `timezone` should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`.
 
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
+* **Syntax:** `TIME_CEIL(timestamp_expr, period[, origin[, timezone]])`
+* **Function type:** Scalar, date and time
 
-Rounds up a timestamp by a given time period, optionally from some reference time or timezone.
+<details><summary>Example</summary>
+
+The following example rounds up the `__time` column from the `taxi-trips` datasource to the nearest 45th minute in reference to the timestamp `2013-08-01 08:0:00`.
+
+```sql
+SELECT 
+  "__time" AS "original_timestamp",
+  TIME_CEIL("__time", 'PT45M', TIMESTAMP '2013-08-01 08:0:00') AS "time_ceiling"
+FROM "taxi-trips"
+LIMIT 2
+```
+
+Returns the following:
+
+| `original_timestamp` | `time_ceiling` |
+| -- | -- |
+| `2013-08-01T08:14:37.000Z` | `2013-08-01T08:45:00.000Z` |
+| `2013-08-01T09:13:00.000Z` | `2013-08-01T09:30:00.000Z` |
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
 
 ## TIME_EXTRACT
 
-`TIME_EXTRACT(<TIMESTAMP>, [<unit>, [<timezone>]])`
+Extracts the value of `unit` from the timestamp and returns it as a number. If provided, `timezone` should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`.
 
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
+* **Syntax:** `TIME_EXTRACT(timestamp_expr[, unit[, timezone]])`
+* **Function type:** Scalar, date and time
 
-Extracts the value of some unit of the timestamp and returns the number.
+<details><summary>Example</summary>
+
+The following example extracts the hour from the `__time` column in the `taxi-trips` datasource and offsets its timezone by `-04:00` hours.
+
+```sql
+SELECT 
+  "__time" AS "original_timestamp",
+  TIME_EXTRACT("__time", 'hour', '-04:00') AS "extract_hour"
+FROM "taxi-trips"
+LIMIT 2
+```
+
+Returns the following:
+
+| `original_timestamp` | `extract_hour` | 
+| -- | -- | 
+| `2013-08-01T08:14:37.000Z` | `4` |
+| `2013-08-01T09:13:00.000Z` | `5` |
+
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
 
 ## TIME_FLOOR
 
-`TIME_FLOOR(<TIMESTAMP>, <period>, [<origin>, [<timezone>]])`
+Rounds down a timestamp to a given ISO 8601 time period. You can specify `origin` to provide a reference timestamp from which to start rounding. If provided, `timezone` should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`.
 
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
+* **Syntax:** `TIME_FLOOR(timestamp_expr, period[, origin[, timezone]])`
+* **Function type:** Scalar, date and time
 
-Rounds down a timestamp by a given time period, optionally from some reference time or timezone.
+<details><summary>Example</summary>
+
+The following example rounds down the `__time` column from the `taxi-trips` datasource to the nearest 45th minute in reference to the timestamp `2013-08-01 08:0:00`.
+
+```sql
+SELECT 
+  "__time" AS "original_timestamp",
+  TIME_FLOOR("__time", 'PT45M', TIMESTAMP '2013-08-01 08:0:00') AS "time_floor"
+FROM "taxi-trips"
+LIMIT 2
+```
+
+Returns the following:
+
+| `original_timestamp` | `time_floor` |
+| -- | -- |
+| `2013-08-01T08:14:37.000Z` | `2013-08-01T08:00:00.000Z` |
+| `2013-08-01T09:13:00.000Z` | `2013-08-01T08:45:00.000Z` |
+
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
 
 ## TIME_FORMAT
 
-`TIME_FORMAT(<TIMESTAMP>, [<pattern>, [<timezone>]])`
+Formats a timestamp as a string in a provided [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html). If no pattern is provided, `pattern` defaults to ISO 8601. If provided, `timezone` should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`.
 
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
+* **Syntax:** `TIME_FORMAT(timestamp_expr[, pattern[, timezone]])`
+* **Function type:** Scalar, date and time
 
-Formats a timestamp as a string.
+<details><summary>Example</summary>
+
+The following example formats the `__time` column from the `flight-carriers` datasource into a string format and offsets the result's timezone by `-05:00` hours.
+
+```sql
+SELECT
+  "__time" AS "original_time",
+TIME_FORMAT( "__time", 'dd-MM-YYYY hh:mm aa zzz', '-05:00') AS "string"
+FROM "taxi-trips"
+LIMIT 1
+```
+
+Returns the following:
+
+| `original_time` | `string` |
+| -- | -- |
+| `2013-08-01T08:14:37.000Z` | `01-08-2013 03:14 AM -05:00` |
+
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
 
 ## TIME_IN_INTERVAL
 
@@ -2663,19 +2806,62 @@ Returns whether a timestamp is contained within a particular interval, formatted
 
 ## TIME_PARSE
 
-`TIME_PARSE(<string_expr>, [<pattern>, [<timezone>]])`
+Parses a string into a timestamp using a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html). If no pattern is provided, `pattern` defaults to ISO 8601. Returns NULL if string cannot be parsed. If provided, `timezone` should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`.
 
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
+* **Syntax:** `TIME_PARSE(string_expr[, pattern[, timezone]])`
+* **Function type:** Scalar, date and time
 
-Parses a string into a timestamp.
+<details><summary>Example</summary>
+
+The following example parses the `FlightDate` STRING column from the `flight-carriers` datasource into a valid timestamp with an offset of `-05:00` hours.
+
+```sql
+SELECT
+  "FlightDate" AS "original_string",
+  TIME_PARSE("FlightDate", 'YYYY-MM-dd', '-05:00') AS "timestamp"
+FROM "flight-carriers"
+LIMIT 1
+```
+
+Returns the following:
+
+| `original_string` | `timestamp` | 
+| -- | -- |
+| `2005-11-01` | `2005-11-01T05:00:00.000Z` |
+
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
 
 ## TIME_SHIFT
 
-`TIME_SHIFT(<TIMESTAMP>, <period>, <step>, [<timezone>])`
+Shifts a timestamp by a given number of time units. The `period` parameter can be any ISO 8601 period. The `step` parameter can be negative. If provided, `timezone` should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`.
 
-**Function type:** [Scalar, date and time](sql-scalar.md#date-and-time-functions)
+* **Syntax:** `TIME_SHIFT(timestamp_expr, period, step[, timezone])`
+* **Function type:** Scalar, date and time
+
+<details><summary>Example</summary>
+
+The following example shifts the `__time` column from the `taxi-trips` datasource back by 24 hours.
+
+```sql
+SELECT
+  "__time" AS "original_timestamp",
+  TIME_SHIFT("__time", 'PT1H', -24) AS "shift_back"
+FROM "taxi-trips"
+LIMIT 1
+```
+
+Returns the following:
+
+| `original_timestamp` | `shift_back` |
+| -- | -- |
+| `2013-08-01T08:14:37.000Z` | `2013-07-31T08:14:37.000Z` | 
+
+</details>
+
+[Learn more](sql-scalar.md#date-and-time-functions)
 
-Shifts a timestamp forwards or backwards by a given number of time units.
 
 ## TIMESTAMP_TO_MILLIS
 
diff --git a/docs/querying/sql-scalar.md b/docs/querying/sql-scalar.md
index ee94ca769da..28e889b8887 100644
--- a/docs/querying/sql-scalar.md
+++ b/docs/querying/sql-scalar.md
@@ -143,11 +143,11 @@ Literal timestamps in the connection time zone can be written using `TIMESTAMP '
 simplest way to write literal timestamps in other time zones is to use TIME_PARSE, like
 `TIME_PARSE('2000-02-01 00:00:00', NULL, 'America/Los_Angeles')`.
 
-The best ways to filter based on time are by using ISO8601 intervals, like
+The best way to filter based on time is by using ISO 8601 intervals, like
 `TIME_IN_INTERVAL(__time, '2000-01-01/2000-02-01')`, or by using literal timestamps with the `>=` and `<` operators, like
 `__time >= TIMESTAMP '2000-01-01 00:00:00' AND __time < TIMESTAMP '2000-02-01 00:00:00'`.
 
-Druid supports the standard SQL BETWEEN operator, but we recommend avoiding it for time filters. BETWEEN is inclusive
+Druid supports the standard SQL `BETWEEN` operator, but we recommend avoiding it for time filters. `BETWEEN` is inclusive
 of its upper bound, which makes it awkward to write time filters correctly. For example, the equivalent of
 `TIME_IN_INTERVAL(__time, '2000-01-01/2000-02-01')` is
 `__time BETWEEN TIMESTAMP '2000-01-01 00:00:00' AND TIMESTAMP '2000-01-31 23:59:59.999'`.
@@ -160,16 +160,16 @@ overhead.
 
 |Function|Notes|
 |--------|-----|
-|`CURRENT_TIMESTAMP`|Current timestamp in the connection's time zone.|
-|`CURRENT_DATE`|Current date in the connection's time zone.|
+|`CURRENT_TIMESTAMP`|Current timestamp in UTC time, unless you specify a different timezone in the query context.|
+|`CURRENT_DATE`|Current date in UTC time, unless you specify a different timezone in the query context.|
 |`DATE_TRUNC(unit, timestamp_expr)`|Rounds down a timestamp, returning it as a new timestamp. Unit can be 'milliseconds', 'second', 'minute', 'hour', 'day', 'week', 'month', 'quarter', 'year', 'decade', 'century', or 'millennium'.|
-|`TIME_CEIL(timestamp_expr, period[, origin[, timezone]])`|Rounds up a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). Specify `origin` as a timestamp to set the reference time for rounding. For example, `TIME_CEIL(__time, 'PT1H', TIMESTAMP '2016-06-27 00:30:00')` measures an hourly period from 00:30-01:30 instead of 00:00-01:00. See [Period granularities](granularities.md) for details on the default starting boundaries. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". This function is similar to `CEIL` but is more flexible.|
-|`TIME_FLOOR(timestamp_expr, period[, origin[, timezone]])`|Rounds down a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). Specify `origin` as a timestamp to set the reference time for rounding. For example, `TIME_FLOOR(__time, 'PT1H', TIMESTAMP '2016-06-27 00:30:00')` measures an hourly period from 00:30-01:30 instead of 00:00-01:00. See [Period granularities](granularities.md) for details on the default starting boundaries. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". This function is similar to `FLOOR` but is more flexible.|
-|`TIME_SHIFT(timestamp_expr, period, step[, timezone])`|Shifts a timestamp by a period (step times), returning it as a new timestamp. Period can be any ISO8601 period. Step may be negative. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|
-|`TIME_EXTRACT(timestamp_expr[, unit[, timezone]])`|Extracts a time part from `expr`, returning it as a number. Unit can be EPOCH, SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". This function is similar to `EXTRACT` but is more flexible. Unit and time zone must be literals, and must be provided quoted, like `TIME_EXTRACT(__time, 'HOUR')` or `TIME_EXTRACT(__time, 'HOUR', 'America/Los_Angeles')`.|
-|`TIME_PARSE(string_expr[, pattern[, timezone]])`|Parses a string into a timestamp using a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 (e.g. `2000-01-02T03:04:05Z`) if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00", and will be used as the time zone for strings that do not include a time zone offset. Pattern and time zone must be literals. Strings that cannot be parsed as timestamps will be returned as NULL.|
-|`TIME_FORMAT(timestamp_expr[, pattern[, timezone]])`|Formats a timestamp as a string with a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 (e.g. `2000-01-02T03:04:05Z`) if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". Pattern and time zone must be literals.|
-|`TIME_IN_INTERVAL(timestamp_expr, interval)`|Returns whether a timestamp is contained within a particular interval. The interval must be a literal string containing any ISO8601 interval, such as `'2001-01-01/P1D'` or `'2001-01-01T01:00:00/2001-01-02T01:00:00'`. The start instant of the interval is inclusive and the end instant is exclusive.|
+|`TIME_CEIL(timestamp_expr, period[, origin[, timezone]])`|Rounds up a timestamp, returning it as a new timestamp. Period can be any ISO 8601 period, like P3M (quarters) or PT12H (half-days). Specify `origin` as a timestamp to set the reference time for rounding. For example, `TIME_CEIL(__time, 'PT1H', TIMESTAMP '2016-06-27 00:30:00')` measures an hourly period from 00:30-01:30 instead of 00:00-01:00. See [Period granularities](granularities.md) for details on the default starting boundaries. The time zone, if provided, should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`. This function is similar to `CEIL` but is more flexible.|
+|`TIME_FLOOR(timestamp_expr, period[, origin[, timezone]])`|Rounds down a timestamp, returning it as a new timestamp. Period can be any ISO 8601 period, like P3M (quarters) or PT12H (half-days). Specify `origin` as a timestamp to set the reference time for rounding. For example, `TIME_FLOOR(__time, 'PT1H', TIMESTAMP '2016-06-27 00:30:00')` measures an hourly period from 00:30-01:30 instead of 00:00-01:00. See [Period granularities](granularities.md) for details on the default starting boundaries. The time zone, if provided, should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`. This function is similar to `FLOOR` but is more flexible.|
+|`TIME_SHIFT(timestamp_expr, period, step[, timezone])`|Shifts a timestamp by a period (step times), returning it as a new timestamp. The `period` parameter can be any ISO 8601 period. The `step` parameter can be negative. The time zone, if provided, should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`.|
+|`TIME_EXTRACT(timestamp_expr, unit[, timezone])`|Extracts a time part from `expr`, returning it as a number. Unit can be EPOCH, SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`. The `unit` and `timezone` parameters must be provided as quoted literals, such as `TIME_EXTRACT(__time, 'HOUR')` or `TIME_EXTRACT(__time, 'HOUR', 'America/Los_Angeles')`. This function is similar to `EXTRACT` but is more flexible. |
+|`TIME_PARSE(string_expr[, pattern[, timezone]])`|Parses a string into a timestamp using a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO 8601 (e.g. `2000-01-02T03:04:05Z`) if the pattern is not provided. The `timezone` parameter is used as the time zone for strings that do not already include a time zone offset. If provided, `timezone` should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`. The `pattern` and `timezone` parameters must be literals. Strings that cannot be parsed as timestamps return NULL.|
+|`TIME_FORMAT(timestamp_expr[, pattern[, timezone]])`|Formats a timestamp as a string with a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO 8601 (e.g. `2000-01-02T03:04:05Z`) if the pattern is not provided. If provided, the `timezone` parameter should be a time zone name like `America/Los_Angeles` or an offset like `-08:00`. The `pattern` and `timezone` parameters must be literals.|
+|`TIME_IN_INTERVAL(timestamp_expr, interval)`|Returns whether a timestamp is contained within a particular interval. The interval must be a literal string containing any ISO 8601 interval, such as `'2001-01-01/P1D'` or `'2001-01-01T01:00:00/2001-01-02T01:00:00'`. The start instant of the interval is inclusive and the end instant is exclusive.|
 |`MILLIS_TO_TIMESTAMP(millis_expr)`|Converts a number of milliseconds since the epoch (1970-01-01 00:00:00 UTC) into a timestamp.|
 |`TIMESTAMP_TO_MILLIS(timestamp_expr)`|Converts a timestamp into a number of milliseconds since the epoch.|
 |`EXTRACT(unit FROM timestamp_expr)`|Extracts a time part from `expr`, returning it as a number. Unit can be EPOCH, MICROSECOND, MILLISECOND, SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), ISODOW (ISO day of week), DOY (day of year), WEEK (week of year), MONTH, QUARTER, YEAR, ISOYEAR, DECADE, CENTURY or MILLENNIUM. Units must be provided unquoted, like `EXTRACT(HOUR FROM __time)`.|

From e2516d9a674ce49342ac80e7509f1ccf5375a1c0 Mon Sep 17 00:00:00 2001
From: Adarsh Sanjeev <adarshsanjeev@gmail.com>
Date: Fri, 23 Aug 2024 11:32:30 +0530
Subject: [PATCH 47/99] WriteOutBytes improvements

This PR generally improves the working of WriteOutBytes and WriteOutMedium. Some analysis of usage of TmpFileSegmentWriteOutMedium shows that they periodically get used for very small things. The overhead of creating a tmp file is actually very large. To improve the performance in these cases, this PR modifies TmpFileSegmentWriteOutMedium to return a heap-based WriteOutBytes that falls back to making a tmp file when it actually fills up.
---------
Co-authored-by: imply-cheddar <eric.tschetter@imply.io>
---
 .../apache/druid/error/DruidException.java    |   2 +-
 .../segment/writeout/FileWriteOutBytes.java   |  53 ++--
 .../LazilyAllocatingHeapWriteOutBytes.java    | 259 ++++++++++++++++++
 .../writeout/LegacyFileWriteOutBytes.java     | 162 +++++++++++
 .../LegacyTmpFileSegmentWriteOutMedium.java   |  76 +++++
 ...cyTmpFileSegmentWriteOutMediumFactory.java |  46 ++++
 .../writeout/SegmentWriteOutMedium.java       |   5 +-
 .../SegmentWriteOutMediumFactory.java         |   2 +
 .../TmpFileSegmentWriteOutMedium.java         |  93 ++++++-
 .../druid/segment/writeout/WriteOutBytes.java |   8 +-
 .../writeout/FileWriteOutBytesTest.java       |  42 ++-
 ...LazilyAllocatingHeapWriteOutBytesTest.java |  77 ++++++
 .../writeout/LegacyFileWriteOutBytesTest.java | 211 ++++++++++++++
 .../TmpFileSegmentWriteOutMediumTest.java     |  80 ++++++
 .../segment/writeout/WriteOutBytesTest.java   |  10 +
 15 files changed, 1068 insertions(+), 58 deletions(-)
 create mode 100644 processing/src/main/java/org/apache/druid/segment/writeout/LazilyAllocatingHeapWriteOutBytes.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/writeout/LegacyFileWriteOutBytes.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/writeout/LegacyTmpFileSegmentWriteOutMedium.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/writeout/LegacyTmpFileSegmentWriteOutMediumFactory.java
 create mode 100644 processing/src/test/java/org/apache/druid/segment/writeout/LazilyAllocatingHeapWriteOutBytesTest.java
 create mode 100644 processing/src/test/java/org/apache/druid/segment/writeout/LegacyFileWriteOutBytesTest.java
 create mode 100644 processing/src/test/java/org/apache/druid/segment/writeout/TmpFileSegmentWriteOutMediumTest.java

diff --git a/processing/src/main/java/org/apache/druid/error/DruidException.java b/processing/src/main/java/org/apache/druid/error/DruidException.java
index b1056bbef52..66190d13a91 100644
--- a/processing/src/main/java/org/apache/druid/error/DruidException.java
+++ b/processing/src/main/java/org/apache/druid/error/DruidException.java
@@ -170,7 +170,7 @@ public class DruidException extends RuntimeException
 
   /**
    * Build a "defensive" exception, this is an exception that should never actually be triggered, but we are
-   * throwing it inside of a defensive check.
+   * throwing it inside a defensive check.
    *
    * @return A builder for a defensive exception.
    */
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/FileWriteOutBytes.java b/processing/src/main/java/org/apache/druid/segment/writeout/FileWriteOutBytes.java
index e41a362fda8..0d0a7a40555 100644
--- a/processing/src/main/java/org/apache/druid/segment/writeout/FileWriteOutBytes.java
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/FileWriteOutBytes.java
@@ -21,8 +21,10 @@ package org.apache.druid.segment.writeout;
 
 import com.google.common.io.ByteStreams;
 import org.apache.druid.io.Channels;
+import org.apache.druid.java.util.common.ByteBufferUtils;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.IOE;
+import org.apache.druid.java.util.common.io.Closer;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -33,22 +35,28 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.WritableByteChannel;
 
-final class FileWriteOutBytes extends WriteOutBytes
+public final class FileWriteOutBytes extends WriteOutBytes
 {
   private final File file;
   private final FileChannel ch;
   private long writeOutBytes;
 
-  /** Purposely big-endian, for {@link #writeInt(int)} implementation */
-  private final ByteBuffer buffer = ByteBuffer.allocate(4096); // 4K page sized buffer
+  /**
+   * Purposely big-endian, for {@link #writeInt(int)} implementation.
+   * Direct because there is a material difference in performance when writing direct buffers
+   */
+  private final ByteBuffer buffer = ByteBuffer.allocateDirect(32768); // 32K page sized buffer
 
-  FileWriteOutBytes(File file, FileChannel ch)
+  FileWriteOutBytes(File file, FileChannel ch, Closer closer)
   {
     this.file = file;
     this.ch = ch;
     this.writeOutBytes = 0L;
+    closer.register(
+        () -> ByteBufferUtils.free(buffer)
+    );
   }
-  
+
   private void flushIfNeeded(int bytesNeeded) throws IOException
   {
     if (buffer.remaining() < bytesNeeded) {
@@ -90,22 +98,29 @@ final class FileWriteOutBytes extends WriteOutBytes
   {
     int len = src.remaining();
     flushIfNeeded(len);
-    while (src.remaining() > buffer.capacity()) {
-      int srcLimit = src.limit();
-      try {
-        src.limit(src.position() + buffer.capacity());
-        buffer.put(src);
-        writeOutBytes += buffer.capacity();
-        flush();
-      }
-      finally {
-        // IOException may occur in flush(), reset src limit to the original
-        src.limit(srcLimit);
+    if (len > buffer.remaining()) {
+      // if a flush was required, flushIfNeeded should have forced a flush.  So, if the len is greater than
+      // our buffer size, we should just dump it straight to the file instead of buffering
+      Channels.writeFully(ch, src);
+      writeOutBytes += len;
+    } else {
+      while (src.remaining() > buffer.capacity()) {
+        int srcLimit = src.limit();
+        try {
+          src.limit(src.position() + buffer.capacity());
+          buffer.put(src);
+          writeOutBytes += buffer.capacity();
+          flush();
+        }
+        finally {
+          // IOException may occur in flush(), reset src limit to the original
+          src.limit(srcLimit);
+        }
       }
+      int remaining = src.remaining();
+      buffer.put(src);
+      writeOutBytes += remaining;
     }
-    int remaining = src.remaining();
-    buffer.put(src);
-    writeOutBytes += remaining;
     return len;
   }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/LazilyAllocatingHeapWriteOutBytes.java b/processing/src/main/java/org/apache/druid/segment/writeout/LazilyAllocatingHeapWriteOutBytes.java
new file mode 100644
index 00000000000..8d3d7bb7612
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/LazilyAllocatingHeapWriteOutBytes.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.writeout;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.input.NullInputStream;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.io.ByteBufferInputStream;
+import org.apache.druid.java.util.common.io.Closer;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.BufferOverflowException;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.function.Supplier;
+
+/**
+ * Lazily decides to use a tmpBuffer to act as WriteOutBytes, till more than certain threshold is reached. Once this is
+ * met, it switches to delegating all calls to a {@link WriteOutBytes} created by delegateSupplier.
+ * <p>
+ * This is useful if the data stored in the {@link WriteOutBytes} is small enough that buffering the changes in memory
+ * would be faster than creating some {@link WriteOutBytes}.
+ */
+public class LazilyAllocatingHeapWriteOutBytes extends WriteOutBytes
+{
+  private final Supplier<WriteOutBytes> delegateSupplier;
+
+  private ByteBuffer tmpBuffer = null;
+  private WriteOutBytes delegate = null;
+  private boolean open = true;
+
+  public LazilyAllocatingHeapWriteOutBytes(Supplier<WriteOutBytes> delegateSupplier, Closer closer)
+  {
+    this.delegateSupplier = delegateSupplier;
+    closer.register(() -> {
+      open = false;
+      tmpBuffer = null;
+      delegate = null;
+    });
+  }
+
+  @Override
+  public void writeInt(int v) throws IOException
+  {
+    checkOpen();
+    final boolean useBuffer = ensureBytes(Integer.BYTES);
+    if (useBuffer) {
+      tmpBuffer.putInt(v);
+      return;
+    }
+
+    delegate.writeInt(v);
+  }
+
+  @Override
+  public long size()
+  {
+    if (delegate == null) {
+      return tmpBuffer == null ? 0 : tmpBuffer.position();
+    } else {
+      return delegate.size();
+    }
+  }
+
+  @Override
+  public void writeTo(WritableByteChannel channel) throws IOException
+  {
+    checkOpen();
+    if (delegate == null) {
+      if (tmpBuffer == null) {
+        return;
+      }
+
+      final ByteBuffer tmpBufCopy = tmpBuffer.asReadOnlyBuffer();
+      tmpBufCopy.flip();
+      channel.write(tmpBufCopy);
+    } else {
+      delegate.writeTo(channel);
+    }
+  }
+
+  @Override
+  public InputStream asInputStream() throws IOException
+  {
+    checkOpen();
+    if (delegate == null) {
+      if (tmpBuffer == null) {
+        return new NullInputStream();
+      }
+      final ByteBuffer tmpBufCopy = tmpBuffer.asReadOnlyBuffer();
+      tmpBufCopy.flip();
+      return new ByteBufferInputStream(tmpBufCopy);
+    } else {
+      return delegate.asInputStream();
+    }
+  }
+
+  @Override
+  public void readFully(long pos, ByteBuffer buffer) throws IOException
+  {
+    checkOpen();
+    if (delegate == null) {
+      if (tmpBuffer == null) {
+        return;
+      }
+
+      if (pos <= tmpBuffer.position()) {
+        final ByteBuffer tmpBufCopy = tmpBuffer.asReadOnlyBuffer();
+        tmpBufCopy.flip().position((int) pos);
+        if (tmpBufCopy.remaining() < buffer.remaining()) {
+          throw new BufferUnderflowException();
+        }
+        tmpBufCopy.limit(tmpBufCopy.position() + buffer.remaining());
+        buffer.put(tmpBufCopy);
+      } else {
+        throw new BufferOverflowException();
+      }
+    } else {
+      delegate.readFully(pos, buffer);
+    }
+  }
+
+  @Override
+  public void write(int b) throws IOException
+  {
+    checkOpen();
+    final boolean useBuffer = ensureBytes(1);
+    if (useBuffer) {
+      tmpBuffer.put((byte) b);
+      return;
+    }
+
+    delegate.write(b);
+  }
+
+  @Override
+  public void write(byte[] b, int off, int len) throws IOException
+  {
+    write(ByteBuffer.wrap(b, off, len));
+  }
+
+  @Override
+  public int write(ByteBuffer src) throws IOException
+  {
+    checkOpen();
+    final int numToWrite = src.remaining();
+    final boolean useBuffer = ensureBytes(numToWrite);
+    if (useBuffer) {
+      tmpBuffer.put(src);
+      return numToWrite;
+    }
+
+    return delegate.write(src);
+  }
+
+  @Override
+  public boolean isOpen()
+  {
+    return open;
+  }
+
+  private void checkOpen()
+  {
+    if (!isOpen()) {
+      throw DruidException.defensive("WriteOutBytes is already closed.");
+    }
+  }
+
+  /**
+   * Ensures bytes are available, returns a boolean indicating if the buffer should be used or the delegate
+   *
+   * @param numBytes the number of bytes that need writing
+   * @return boolean true if the buffer should be used, false if the delegate should be used
+   * @throws IOException if an issue with IO occurs (can primarily happen when copying buffers)
+   */
+  private boolean ensureBytes(int numBytes) throws IOException
+  {
+    if (tmpBuffer == null) {
+      if (delegate == null) {
+        if (numBytes < 128) {
+          tmpBuffer = ByteBuffer.allocate(128);
+        } else if (numBytes < 4096) {
+          tmpBuffer = ByteBuffer.allocate(4096);
+        } else {
+          // We are likely dealing with something that's already buffering stuff, so just switch delegate immediately
+          delegate = delegateSupplier.get();
+          return false;
+        }
+        return true;
+      } else {
+        return false;
+      }
+    }
+
+    if (numBytes < tmpBuffer.remaining()) {
+      return true;
+    }
+
+    final ByteBuffer newBuf;
+    switch (tmpBuffer.capacity()) {
+      case 128:
+        if (numBytes < 4096 - tmpBuffer.position()) {
+          newBuf = ByteBuffer.allocate(4096);
+          break;
+        }
+      case 4096:
+        if (numBytes < 16384 - tmpBuffer.position()) {
+          newBuf = ByteBuffer.allocate(16384);
+          break;
+        }
+      default:
+        newBuf = null;
+    }
+
+    if (newBuf == null) {
+      delegate = delegateSupplier.get();
+      tmpBuffer.flip();
+      delegate.write(tmpBuffer);
+      tmpBuffer = null;
+      return false;
+    } else {
+      tmpBuffer.flip();
+      newBuf.put(tmpBuffer);
+      tmpBuffer = newBuf;
+      return true;
+    }
+  }
+
+  @VisibleForTesting
+  ByteBuffer getTmpBuffer()
+  {
+    return tmpBuffer;
+  }
+
+  @VisibleForTesting
+  WriteOutBytes getDelegate()
+  {
+    return delegate;
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/LegacyFileWriteOutBytes.java b/processing/src/main/java/org/apache/druid/segment/writeout/LegacyFileWriteOutBytes.java
new file mode 100644
index 00000000000..192aa9b61c3
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/LegacyFileWriteOutBytes.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.writeout;
+
+import com.google.common.io.ByteStreams;
+import org.apache.druid.io.Channels;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.IOE;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
+
+final class LegacyFileWriteOutBytes extends WriteOutBytes
+{
+  private final File file;
+  private final FileChannel ch;
+  private long writeOutBytes;
+
+  /** Purposely big-endian, for {@link #writeInt(int)} implementation */
+  private final ByteBuffer buffer = ByteBuffer.allocate(4096); // 4K page sized buffer
+
+  LegacyFileWriteOutBytes(File file, FileChannel ch)
+  {
+    this.file = file;
+    this.ch = ch;
+    this.writeOutBytes = 0L;
+  }
+
+  private void flushIfNeeded(int bytesNeeded) throws IOException
+  {
+    if (buffer.remaining() < bytesNeeded) {
+      flush();
+    }
+  }
+
+  @Override
+  public void flush() throws IOException
+  {
+    buffer.flip();
+    try {
+      Channels.writeFully(ch, buffer);
+    }
+    catch (IOException e) {
+      throw new IOE(e, "Failed to write to file: %s. Current size of file: %d", file.getAbsolutePath(), writeOutBytes);
+    }
+    buffer.clear();
+  }
+
+  @Override
+  public void write(int b) throws IOException
+  {
+    flushIfNeeded(1);
+    buffer.put((byte) b);
+    writeOutBytes++;
+  }
+
+  @Override
+  public void writeInt(int v) throws IOException
+  {
+    flushIfNeeded(Integer.BYTES);
+    buffer.putInt(v);
+    writeOutBytes += Integer.BYTES;
+  }
+
+  @Override
+  public int write(ByteBuffer src) throws IOException
+  {
+    int len = src.remaining();
+    flushIfNeeded(len);
+    while (src.remaining() > buffer.capacity()) {
+      int srcLimit = src.limit();
+      try {
+        src.limit(src.position() + buffer.capacity());
+        buffer.put(src);
+        writeOutBytes += buffer.capacity();
+        flush();
+      }
+      finally {
+        // IOException may occur in flush(), reset src limit to the original
+        src.limit(srcLimit);
+      }
+    }
+    int remaining = src.remaining();
+    buffer.put(src);
+    writeOutBytes += remaining;
+    return len;
+  }
+
+  @Override
+  public void write(byte[] b, int off, int len) throws IOException
+  {
+    write(ByteBuffer.wrap(b, off, len));
+  }
+
+  @Override
+  public long size()
+  {
+    return writeOutBytes;
+  }
+
+  @Override
+  public void writeTo(WritableByteChannel channel) throws IOException
+  {
+    flush();
+    ch.position(0);
+    try {
+      ByteStreams.copy(ch, channel);
+    }
+    finally {
+      ch.position(ch.size());
+    }
+  }
+
+  @Override
+  public void readFully(long pos, ByteBuffer buffer) throws IOException
+  {
+    if (pos < 0 || pos > writeOutBytes) {
+      throw new IAE("pos %d out of range [%d, %d]", pos, 0, writeOutBytes);
+    }
+    flush();
+    ch.read(buffer, pos);
+    if (buffer.remaining() > 0) {
+      throw new BufferUnderflowException();
+    }
+  }
+
+  @Override
+  public InputStream asInputStream() throws IOException
+  {
+    flush();
+    return new FileInputStream(file);
+  }
+
+  @Override
+  public boolean isOpen()
+  {
+    return ch.isOpen();
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/LegacyTmpFileSegmentWriteOutMedium.java b/processing/src/main/java/org/apache/druid/segment/writeout/LegacyTmpFileSegmentWriteOutMedium.java
new file mode 100644
index 00000000000..a01280bf18d
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/LegacyTmpFileSegmentWriteOutMedium.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.writeout;
+
+import org.apache.druid.java.util.common.FileUtils;
+import org.apache.druid.java.util.common.io.Closer;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
+
+public final class LegacyTmpFileSegmentWriteOutMedium implements SegmentWriteOutMedium
+{
+  private final File dir;
+  private final Closer closer = Closer.create();
+
+  LegacyTmpFileSegmentWriteOutMedium(File outDir) throws IOException
+  {
+    File tmpOutputFilesDir = new File(outDir, "tmpOutputFiles");
+    FileUtils.mkdirp(tmpOutputFilesDir);
+    closer.register(() -> FileUtils.deleteDirectory(tmpOutputFilesDir));
+    this.dir = tmpOutputFilesDir;
+  }
+
+  @Override
+  public WriteOutBytes makeWriteOutBytes() throws IOException
+  {
+    File file = File.createTempFile("filePeon", null, dir);
+    FileChannel ch = FileChannel.open(
+        file.toPath(),
+        StandardOpenOption.READ,
+        StandardOpenOption.WRITE
+    );
+    closer.register(file::delete);
+    closer.register(ch);
+    return new LegacyFileWriteOutBytes(file, ch);
+  }
+
+  @Override
+  public SegmentWriteOutMedium makeChildWriteOutMedium() throws IOException
+  {
+    LegacyTmpFileSegmentWriteOutMedium tmpFileSegmentWriteOutMedium = new LegacyTmpFileSegmentWriteOutMedium(dir);
+    closer.register(tmpFileSegmentWriteOutMedium);
+    return tmpFileSegmentWriteOutMedium;
+  }
+
+  @Override
+  public Closer getCloser()
+  {
+    return closer;
+  }
+
+  @Override
+  public void close() throws IOException
+  {
+    closer.close();
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/LegacyTmpFileSegmentWriteOutMediumFactory.java b/processing/src/main/java/org/apache/druid/segment/writeout/LegacyTmpFileSegmentWriteOutMediumFactory.java
new file mode 100644
index 00000000000..c89627401dc
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/LegacyTmpFileSegmentWriteOutMediumFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.writeout;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+import java.io.File;
+import java.io.IOException;
+
+public final class LegacyTmpFileSegmentWriteOutMediumFactory implements SegmentWriteOutMediumFactory
+{
+  private static final LegacyTmpFileSegmentWriteOutMediumFactory INSTANCE = new LegacyTmpFileSegmentWriteOutMediumFactory();
+
+  @JsonCreator
+  public static LegacyTmpFileSegmentWriteOutMediumFactory instance()
+  {
+    return INSTANCE;
+  }
+
+  private LegacyTmpFileSegmentWriteOutMediumFactory()
+  {
+  }
+
+  @Override
+  public SegmentWriteOutMedium makeSegmentWriteOutMedium(File outDir) throws IOException
+  {
+    return new LegacyTmpFileSegmentWriteOutMedium(outDir);
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/SegmentWriteOutMedium.java b/processing/src/main/java/org/apache/druid/segment/writeout/SegmentWriteOutMedium.java
index 7c2af89b1b6..16efd07cb1b 100644
--- a/processing/src/main/java/org/apache/druid/segment/writeout/SegmentWriteOutMedium.java
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/SegmentWriteOutMedium.java
@@ -28,7 +28,7 @@ import java.io.IOException;
  * SegmentWriteOutMedium is an umbrella "resource disposer" for temporary buffers (in the form of {@link WriteOutBytes},
  * obtained by calling {@link #makeWriteOutBytes()} on the SegmentWriteOutMedium instance), that are used during new Druid
  * segment creation, and other resources (see {@link #getCloser()}).
- *
+ * <p>
  * When SegmentWriteOutMedium is closed, all child WriteOutBytes couldn't be used anymore.
  */
 public interface SegmentWriteOutMedium extends Closeable
@@ -37,6 +37,7 @@ public interface SegmentWriteOutMedium extends Closeable
    * Creates a new empty {@link WriteOutBytes}, attached to this SegmentWriteOutMedium. When this SegmentWriteOutMedium is
    * closed, the returned WriteOutBytes couldn't be used anymore.
    */
+  @SuppressWarnings("RedundantThrows")
   WriteOutBytes makeWriteOutBytes() throws IOException;
 
   /**
@@ -45,7 +46,7 @@ public interface SegmentWriteOutMedium extends Closeable
    * using a shared {@link SegmentWriteOutMedium} but which control the complete lifecycle of the {@link WriteOutBytes}
    * which they require to free the backing resources when they are finished, rather than waiting until
    * {@link #close()} is called for this medium.
-   *
+   * <p>
    * The 'child' medium will be closed when {@link #close()} is called, if not called explicitly prior to closing this
    * medium.
    */
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/SegmentWriteOutMediumFactory.java b/processing/src/main/java/org/apache/druid/segment/writeout/SegmentWriteOutMediumFactory.java
index ae241b3187b..3e34af17f39 100644
--- a/processing/src/main/java/org/apache/druid/segment/writeout/SegmentWriteOutMediumFactory.java
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/SegmentWriteOutMediumFactory.java
@@ -30,6 +30,7 @@ import java.util.Set;
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = TmpFileSegmentWriteOutMediumFactory.class)
 @JsonSubTypes(value = {
     @JsonSubTypes.Type(name = "tmpFile", value = TmpFileSegmentWriteOutMediumFactory.class),
+    @JsonSubTypes.Type(name = "legacyTmpFile", value = LegacyTmpFileSegmentWriteOutMediumFactory.class),
     @JsonSubTypes.Type(name = "offHeapMemory", value = OffHeapMemorySegmentWriteOutMediumFactory.class),
     @JsonSubTypes.Type(name = "onHeapMemory", value = OnHeapMemorySegmentWriteOutMediumFactory.class),
 })
@@ -39,6 +40,7 @@ public interface SegmentWriteOutMediumFactory
   {
     return ImmutableSet.of(
         TmpFileSegmentWriteOutMediumFactory.instance(),
+        LegacyTmpFileSegmentWriteOutMediumFactory.instance(),
         OffHeapMemorySegmentWriteOutMediumFactory.instance(),
         OnHeapMemorySegmentWriteOutMediumFactory.instance()
     );
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/TmpFileSegmentWriteOutMedium.java b/processing/src/main/java/org/apache/druid/segment/writeout/TmpFileSegmentWriteOutMedium.java
index 007b93aa04b..58dc99198e3 100644
--- a/processing/src/main/java/org/apache/druid/segment/writeout/TmpFileSegmentWriteOutMedium.java
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/TmpFileSegmentWriteOutMedium.java
@@ -19,21 +19,51 @@
 
 package org.apache.druid.segment.writeout;
 
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.java.util.common.logger.Logger;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.file.StandardOpenOption;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
+/**
+ * Builds segment write out medium objects that are based on temporary files.  Some analysis of usage of these
+ * objects shows that they periodically get used for very small things, where the overhead of creating a tmp file
+ * is actually very large.  It would be best to go back and look at usages and try to make them lazy such that
+ * they only actually use a medium when they need it.  But, in an attempt to get some benefits, we "shim" in the
+ * laziness by returning a heap-based WriteOutBytes that only falls back to making a tmp file when it actually
+ * fills up.
+ */
 public final class TmpFileSegmentWriteOutMedium implements SegmentWriteOutMedium
 {
+  private static final Logger log = new Logger(TmpFileSegmentWriteOutMedium.class);
+
+  private final AtomicInteger filesCreated;
+  private final SortedMap<Long, Integer> sizeDistribution;
+  private int numLocallyCreated = 0;
+  private boolean root = false;
+
   private final File dir;
   private final Closer closer = Closer.create();
 
   TmpFileSegmentWriteOutMedium(File outDir) throws IOException
   {
+    this(outDir, new AtomicInteger(0), new ConcurrentSkipListMap<>());
+    root = true;
+  }
+
+  private TmpFileSegmentWriteOutMedium(File outDir, AtomicInteger filesCreated, SortedMap<Long, Integer> sizeDistribution) throws IOException
+  {
+    this.filesCreated = filesCreated;
+    this.sizeDistribution = sizeDistribution;
     File tmpOutputFilesDir = new File(outDir, "tmpOutputFiles");
     FileUtils.mkdirp(tmpOutputFilesDir);
     closer.register(() -> FileUtils.deleteDirectory(tmpOutputFilesDir));
@@ -41,23 +71,44 @@ public final class TmpFileSegmentWriteOutMedium implements SegmentWriteOutMedium
   }
 
   @Override
-  public WriteOutBytes makeWriteOutBytes() throws IOException
+  public WriteOutBytes makeWriteOutBytes()
   {
-    File file = File.createTempFile("filePeon", null, dir);
-    FileChannel ch = FileChannel.open(
-        file.toPath(),
-        StandardOpenOption.READ,
-        StandardOpenOption.WRITE
+    return new LazilyAllocatingHeapWriteOutBytes(
+        () -> {
+          final int i = filesCreated.incrementAndGet();
+          ++numLocallyCreated;
+          File file;
+          FileChannel ch;
+          try {
+            file = File.createTempFile("filePeon", null, dir);
+            ch = FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE);
+          }
+          catch (IOException e) {
+            throw DruidException.defensive(e, "Failed");
+          }
+          if (i % 1000 == 0) {
+            log.info("Created [%,d] tmp files. [%s]", i, dir);
+          }
+          final FileWriteOutBytes retVal = new FileWriteOutBytes(file, ch, closer);
+          closer.register(file::delete);
+          closer.register(ch);
+          closer.register(() -> {
+            sizeDistribution.compute(retVal.size(), (key, val) -> val == null ? 1 : val + 1);
+          });
+          return retVal;
+        },
+        closer
     );
-    closer.register(file::delete);
-    closer.register(ch);
-    return new FileWriteOutBytes(file, ch);
   }
 
   @Override
   public SegmentWriteOutMedium makeChildWriteOutMedium() throws IOException
   {
-    TmpFileSegmentWriteOutMedium tmpFileSegmentWriteOutMedium = new TmpFileSegmentWriteOutMedium(dir);
+    TmpFileSegmentWriteOutMedium tmpFileSegmentWriteOutMedium = new TmpFileSegmentWriteOutMedium(
+        dir,
+        filesCreated,
+        sizeDistribution
+    );
     closer.register(tmpFileSegmentWriteOutMedium);
     return tmpFileSegmentWriteOutMedium;
   }
@@ -71,6 +122,28 @@ public final class TmpFileSegmentWriteOutMedium implements SegmentWriteOutMedium
   @Override
   public void close() throws IOException
   {
+    log.debug("Closing, files still open[%,d], filesBeingClosed[%,d], dir[%s]", filesCreated.get(), numLocallyCreated, dir);
+    filesCreated.set(filesCreated.get() - numLocallyCreated);
+    numLocallyCreated = 0;
     closer.close();
+
+    if (root && log.isDebugEnabled()) {
+      log.debug("Size distribution of files:");
+      for (Map.Entry<Long, Integer> entry : sizeDistribution.entrySet()) {
+        log.debug("%,15d => %,15d", entry.getKey(), entry.getValue());
+      }
+    }
+  }
+
+  @VisibleForTesting
+  int getFilesCreated()
+  {
+    return filesCreated.get();
+  }
+
+  @VisibleForTesting
+  int getNumLocallyCreated()
+  {
+    return numLocallyCreated;
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/writeout/WriteOutBytes.java b/processing/src/main/java/org/apache/druid/segment/writeout/WriteOutBytes.java
index e1c2972b148..95ab2f7b01d 100644
--- a/processing/src/main/java/org/apache/druid/segment/writeout/WriteOutBytes.java
+++ b/processing/src/main/java/org/apache/druid/segment/writeout/WriteOutBytes.java
@@ -30,7 +30,7 @@ import java.nio.channels.WritableByteChannel;
  * WritableByteChannel} and {@link #writeInt(int)} append to the sequence. Methods {@link
  * #writeTo(WritableByteChannel)} and {@link #asInputStream()} allow to write the sequence somewhere else. {@link
  * #readFully} allows to access the sequence randomly.
- *
+ * <p>
  * WriteOutBytes is a resource that is managed by {@link SegmentWriteOutMedium}, so it's own {@link #close()} method
  * does nothing. However WriteOutBytes should appear closed, i. e. {@link #isOpen()} returns false, after the parental
  * SegmentWriteOutMedium is closed.
@@ -47,6 +47,12 @@ public abstract class WriteOutBytes extends OutputStream implements WritableByte
    */
   public abstract long size();
 
+  @Override
+  public void write(byte[] b, int off, int len) throws IOException
+  {
+    write(ByteBuffer.wrap(b, off, len));
+  }
+
   /**
    * Takes all bytes that are written to this WriteOutBytes so far and writes them into the given channel.
    */
diff --git a/processing/src/test/java/org/apache/druid/segment/writeout/FileWriteOutBytesTest.java b/processing/src/test/java/org/apache/druid/segment/writeout/FileWriteOutBytesTest.java
index da6b3c481a5..cd39c76c99d 100644
--- a/processing/src/test/java/org/apache/druid/segment/writeout/FileWriteOutBytesTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/writeout/FileWriteOutBytesTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.segment.writeout;
 
 import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.io.Closer;
 import org.easymock.EasyMock;
 import org.junit.Assert;
 import org.junit.Before;
@@ -41,13 +42,13 @@ public class FileWriteOutBytesTest
   {
     mockFileChannel = EasyMock.mock(FileChannel.class);
     mockFile = EasyMock.mock(File.class);
-    fileWriteOutBytes = new FileWriteOutBytes(mockFile, mockFileChannel);
+    fileWriteOutBytes = new FileWriteOutBytes(mockFile, mockFileChannel, Closer.create());
   }
 
   @Test
-  public void write4KiBIntsShouldNotFlush() throws IOException
+  public void write32KiBIntsShouldNotFlush() throws IOException
   {
-    // Write 4KiB of ints and expect the write operation of the file channel will be triggered only once.
+    // Write 32KiB of ints and expect the write operation of the file channel will be triggered only once.
     EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
             .andAnswer(() -> {
               ByteBuffer buffer = (ByteBuffer) EasyMock.getCurrentArguments()[0];
@@ -56,13 +57,13 @@ public class FileWriteOutBytesTest
               return remaining;
             }).times(1);
     EasyMock.replay(mockFileChannel);
-    final int writeBytes = 4096;
+    final int writeBytes = 32768;
     final int numOfInt = writeBytes / Integer.BYTES;
     for (int i = 0; i < numOfInt; i++) {
       fileWriteOutBytes.writeInt(i);
     }
-    // no need to flush up to 4KiB
-    // the first byte after 4KiB will cause a flush
+    // no need to flush up to 32KiB
+    // the first byte after 32KiB will cause a flush
     fileWriteOutBytes.write(1);
     EasyMock.verify(mockFileChannel);
   }
@@ -92,14 +93,14 @@ public class FileWriteOutBytesTest
               return remaining;
             }).times(1);
     EasyMock.replay(mockFileChannel);
-    ByteBuffer src = ByteBuffer.allocate(4096 + 1);
+    ByteBuffer src = ByteBuffer.allocate(32768 + 1);
     fileWriteOutBytes.write(src);
     Assert.assertEquals(src.capacity(), fileWriteOutBytes.size());
     EasyMock.verify(mockFileChannel);
   }
 
   @Test
-  public void writeBufferLargerThanCapacityThrowsIOEInTheMiddleShouldIncrementSizeCorrectly() throws IOException
+  public void writeBufferLargerThanCapacityWritesDirectlyToFileShouldIncrementSizeCorrectly() throws IOException
   {
     EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
             .andAnswer(() -> {
@@ -108,25 +109,16 @@ public class FileWriteOutBytesTest
               buffer.position(remaining);
               return remaining;
             }).once();
-    EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
-            .andThrow(new IOException())
-            .once();
     EasyMock.replay(mockFileChannel);
-    ByteBuffer src = ByteBuffer.allocate(4096 * 2 + 1);
-    try {
-      fileWriteOutBytes.write(src);
-      Assert.fail("IOException should have been thrown.");
-    }
-    catch (IOException e) {
-      // The second invocation to flush bytes fails. So the size should count what has already been put successfully
-      Assert.assertEquals(4096 * 2, fileWriteOutBytes.size());
-    }
+    ByteBuffer src = ByteBuffer.allocate(32768 * 2 + 1);
+    fileWriteOutBytes.write(src);
+    Assert.assertEquals(32768 * 2 + 1, fileWriteOutBytes.size());
   }
 
   @Test
   public void writeBufferSmallerThanCapacityShouldIncrementSizeCorrectly() throws IOException
   {
-    ByteBuffer src = ByteBuffer.allocate(4096);
+    ByteBuffer src = ByteBuffer.allocate(32768);
     fileWriteOutBytes.write(src);
     Assert.assertEquals(src.capacity(), fileWriteOutBytes.size());
   }
@@ -146,11 +138,11 @@ public class FileWriteOutBytesTest
   @Test
   public void testReadFullyWorks() throws IOException
   {
-    int fileSize = 4096;
+    int fileSize = 32768;
     int numOfInt = fileSize / Integer.BYTES;
     ByteBuffer destination = ByteBuffer.allocate(Integer.BYTES);
     ByteBuffer underlying = ByteBuffer.allocate(fileSize);
-    // Write 4KiB of ints and expect the write operation of the file channel will be triggered only once.
+    // Write 32KiB of ints and expect the write operation of the file channel will be triggered only once.
     EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
             .andAnswer(() -> {
               ByteBuffer buffer = (ByteBuffer) EasyMock.getCurrentArguments()[0];
@@ -181,7 +173,7 @@ public class FileWriteOutBytesTest
   @Test
   public void testReadFullyOutOfBoundsDoesnt() throws IOException
   {
-    int fileSize = 4096;
+    int fileSize = 32768;
     int numOfInt = fileSize / Integer.BYTES;
     ByteBuffer destination = ByteBuffer.allocate(Integer.BYTES);
     EasyMock.replay(mockFileChannel);
@@ -191,7 +183,7 @@ public class FileWriteOutBytesTest
     Assert.assertEquals(fileSize, fileWriteOutBytes.size());
 
     destination.position(0);
-    Assert.assertThrows(IAE.class, () -> fileWriteOutBytes.readFully(5000, destination));
+    Assert.assertThrows(IAE.class, () -> fileWriteOutBytes.readFully(33000, destination));
     EasyMock.verify(mockFileChannel);
   }
 
diff --git a/processing/src/test/java/org/apache/druid/segment/writeout/LazilyAllocatingHeapWriteOutBytesTest.java b/processing/src/test/java/org/apache/druid/segment/writeout/LazilyAllocatingHeapWriteOutBytesTest.java
new file mode 100644
index 00000000000..32a5d09ba00
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/segment/writeout/LazilyAllocatingHeapWriteOutBytesTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.writeout;
+
+import org.apache.druid.java.util.common.io.Closer;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class LazilyAllocatingHeapWriteOutBytesTest
+{
+  private LazilyAllocatingHeapWriteOutBytes target;
+  private Closer closer;
+  private HeapByteBufferWriteOutBytes heapByteBufferWriteOutBytes;
+
+  @Before
+  public void setUp()
+  {
+    closer = Closer.create();
+    heapByteBufferWriteOutBytes = new HeapByteBufferWriteOutBytes();
+    target = new LazilyAllocatingHeapWriteOutBytes(
+        () -> heapByteBufferWriteOutBytes,
+        closer
+    );
+  }
+
+  @Test
+  public void testWritingToBuffer() throws IOException
+  {
+    Assert.assertNull(target.getTmpBuffer());
+
+    target.write(ByteBuffer.allocate(512));
+    Assert.assertNotNull(target.getTmpBuffer());
+    Assert.assertEquals(4096, target.getTmpBuffer().limit());
+    Assert.assertNull(target.getDelegate());
+
+    target.write(ByteBuffer.allocate(16385));
+    Assert.assertNull(target.getTmpBuffer());
+    Assert.assertNotNull(target.getDelegate());
+    Assert.assertEquals(16385 + 512, target.getDelegate().size());
+  }
+
+  @Test
+  public void testClosingWriteOutBytes() throws IOException
+  {
+    Assert.assertNull(target.getTmpBuffer());
+
+    target.writeInt(5);
+    Assert.assertNotNull(target.getTmpBuffer());
+    Assert.assertEquals(128, target.getTmpBuffer().limit());
+    Assert.assertNull(target.getDelegate());
+
+    closer.close();
+
+    Assert.assertNull(target.getTmpBuffer());
+  }
+}
diff --git a/processing/src/test/java/org/apache/druid/segment/writeout/LegacyFileWriteOutBytesTest.java b/processing/src/test/java/org/apache/druid/segment/writeout/LegacyFileWriteOutBytesTest.java
new file mode 100644
index 00000000000..1ab2f5f785b
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/segment/writeout/LegacyFileWriteOutBytesTest.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.writeout;
+
+import org.apache.druid.java.util.common.IAE;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+public class LegacyFileWriteOutBytesTest
+{
+  private LegacyFileWriteOutBytes fileWriteOutBytes;
+  private FileChannel mockFileChannel;
+  private File mockFile;
+
+  @Before
+  public void setUp()
+  {
+    mockFileChannel = EasyMock.mock(FileChannel.class);
+    mockFile = EasyMock.mock(File.class);
+    fileWriteOutBytes = new LegacyFileWriteOutBytes(mockFile, mockFileChannel);
+  }
+
+  @Test
+  public void write4KiBIntsShouldNotFlush() throws IOException
+  {
+    // Write 4KiB of ints and expect the write operation of the file channel will be triggered only once.
+    EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
+            .andAnswer(() -> {
+              ByteBuffer buffer = (ByteBuffer) EasyMock.getCurrentArguments()[0];
+              int remaining = buffer.remaining();
+              buffer.position(remaining);
+              return remaining;
+            }).times(1);
+    EasyMock.replay(mockFileChannel);
+    final int writeBytes = 4096;
+    final int numOfInt = writeBytes / Integer.BYTES;
+    for (int i = 0; i < numOfInt; i++) {
+      fileWriteOutBytes.writeInt(i);
+    }
+    // no need to flush up to 4KiB
+    // the first byte after 4KiB will cause a flush
+    fileWriteOutBytes.write(1);
+    EasyMock.verify(mockFileChannel);
+  }
+
+  @Test
+  public void writeShouldIncrementSize() throws IOException
+  {
+    fileWriteOutBytes.write(1);
+    Assert.assertEquals(1, fileWriteOutBytes.size());
+  }
+
+  @Test
+  public void writeIntShouldIncrementSize() throws IOException
+  {
+    fileWriteOutBytes.writeInt(1);
+    Assert.assertEquals(4, fileWriteOutBytes.size());
+  }
+
+  @Test
+  public void writeBufferLargerThanCapacityShouldIncrementSizeCorrectly() throws IOException
+  {
+    EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
+            .andAnswer(() -> {
+              ByteBuffer buffer = (ByteBuffer) EasyMock.getCurrentArguments()[0];
+              int remaining = buffer.remaining();
+              buffer.position(remaining);
+              return remaining;
+            }).times(1);
+    EasyMock.replay(mockFileChannel);
+    ByteBuffer src = ByteBuffer.allocate(4096 + 1);
+    fileWriteOutBytes.write(src);
+    Assert.assertEquals(src.capacity(), fileWriteOutBytes.size());
+    EasyMock.verify(mockFileChannel);
+  }
+
+  @Test
+  public void writeBufferLargerThanCapacityThrowsIOEInTheMiddleShouldIncrementSizeCorrectly() throws IOException
+  {
+    EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
+            .andAnswer(() -> {
+              ByteBuffer buffer = (ByteBuffer) EasyMock.getCurrentArguments()[0];
+              int remaining = buffer.remaining();
+              buffer.position(remaining);
+              return remaining;
+            }).once();
+    EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
+            .andThrow(new IOException())
+            .once();
+    EasyMock.replay(mockFileChannel);
+    ByteBuffer src = ByteBuffer.allocate(4096 * 2 + 1);
+    try {
+      fileWriteOutBytes.write(src);
+      Assert.fail("IOException should have been thrown.");
+    }
+    catch (IOException e) {
+      // The second invocation to flush bytes fails. So the size should count what has already been put successfully
+      Assert.assertEquals(4096 * 2, fileWriteOutBytes.size());
+    }
+  }
+
+  @Test
+  public void writeBufferSmallerThanCapacityShouldIncrementSizeCorrectly() throws IOException
+  {
+    ByteBuffer src = ByteBuffer.allocate(4096);
+    fileWriteOutBytes.write(src);
+    Assert.assertEquals(src.capacity(), fileWriteOutBytes.size());
+  }
+  @Test
+  public void sizeDoesNotFlush() throws IOException
+  {
+    EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
+            .andThrow(new AssertionError("file channel should not have been written to."));
+    EasyMock.replay(mockFileChannel);
+    long size = fileWriteOutBytes.size();
+    Assert.assertEquals(0, size);
+    fileWriteOutBytes.writeInt(10);
+    size = fileWriteOutBytes.size();
+    Assert.assertEquals(4, size);
+  }
+
+  @Test
+  public void testReadFullyWorks() throws IOException
+  {
+    int fileSize = 4096;
+    int numOfInt = fileSize / Integer.BYTES;
+    ByteBuffer destination = ByteBuffer.allocate(Integer.BYTES);
+    ByteBuffer underlying = ByteBuffer.allocate(fileSize);
+    // Write 4KiB of ints and expect the write operation of the file channel will be triggered only once.
+    EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class)))
+            .andAnswer(() -> {
+              ByteBuffer buffer = (ByteBuffer) EasyMock.getCurrentArguments()[0];
+              underlying.position(0);
+              underlying.put(buffer);
+              return 0;
+            }).times(1);
+    EasyMock.expect(mockFileChannel.read(EasyMock.eq(destination), EasyMock.eq(100L * Integer.BYTES)))
+            .andAnswer(() -> {
+              ByteBuffer buffer = (ByteBuffer) EasyMock.getCurrentArguments()[0];
+              long pos = (long) EasyMock.getCurrentArguments()[1];
+              buffer.putInt(underlying.getInt((int) pos));
+              return Integer.BYTES;
+            }).times(1);
+    EasyMock.replay(mockFileChannel);
+    for (int i = 0; i < numOfInt; i++) {
+      fileWriteOutBytes.writeInt(i);
+    }
+    Assert.assertEquals(underlying.capacity(), fileWriteOutBytes.size());
+
+    destination.position(0);
+    fileWriteOutBytes.readFully(100L * Integer.BYTES, destination);
+    destination.position(0);
+    Assert.assertEquals(100, destination.getInt());
+    EasyMock.verify(mockFileChannel);
+  }
+
+  @Test
+  public void testReadFullyOutOfBoundsDoesnt() throws IOException
+  {
+    int fileSize = 4096;
+    int numOfInt = fileSize / Integer.BYTES;
+    ByteBuffer destination = ByteBuffer.allocate(Integer.BYTES);
+    EasyMock.replay(mockFileChannel);
+    for (int i = 0; i < numOfInt; i++) {
+      fileWriteOutBytes.writeInt(i);
+    }
+    Assert.assertEquals(fileSize, fileWriteOutBytes.size());
+
+    destination.position(0);
+    Assert.assertThrows(IAE.class, () -> fileWriteOutBytes.readFully(5000, destination));
+    EasyMock.verify(mockFileChannel);
+  }
+
+  @Test
+  public void testIOExceptionHasFileInfo() throws Exception
+  {
+    IOException cause = new IOException("Too many bytes");
+    EasyMock.expect(mockFileChannel.write(EasyMock.anyObject(ByteBuffer.class))).andThrow(cause);
+    EasyMock.expect(mockFile.getAbsolutePath()).andReturn("/tmp/file");
+    EasyMock.replay(mockFileChannel, mockFile);
+    fileWriteOutBytes.writeInt(10);
+    fileWriteOutBytes.write(new byte[30]);
+    IOException actual = Assert.assertThrows(IOException.class, () -> fileWriteOutBytes.flush());
+    Assert.assertEquals(String.valueOf(actual.getCause()), actual.getCause(), cause);
+    Assert.assertEquals(actual.getMessage(), actual.getMessage(), "Failed to write to file: /tmp/file. Current size of file: 34");
+  }
+}
diff --git a/processing/src/test/java/org/apache/druid/segment/writeout/TmpFileSegmentWriteOutMediumTest.java b/processing/src/test/java/org/apache/druid/segment/writeout/TmpFileSegmentWriteOutMediumTest.java
new file mode 100644
index 00000000000..11b1acdda24
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/segment/writeout/TmpFileSegmentWriteOutMediumTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.writeout;
+
+import org.apache.druid.java.util.common.FileUtils;
+import org.apache.druid.java.util.common.concurrent.Execs;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class TmpFileSegmentWriteOutMediumTest
+{
+  private volatile TmpFileSegmentWriteOutMedium writeOutMedium;
+  private ExecutorService executorService;
+
+  @Before
+  public void setUp() throws IOException
+  {
+    writeOutMedium = new TmpFileSegmentWriteOutMedium(FileUtils.createTempDir());
+    executorService = Execs.multiThreaded(3, "writeOutMedium-%d");
+  }
+
+  @After
+  public void tearDown()
+  {
+    executorService.shutdownNow();
+  }
+
+  @Test
+  public void testFileCount() throws InterruptedException
+  {
+    final int threadCount = 3;
+    final CountDownLatch latch = new CountDownLatch(threadCount);
+
+    for (int i = 0; i < threadCount; i++) {
+      executorService.submit(
+          () -> {
+            WriteOutBytes writeOutBytes = writeOutMedium.makeWriteOutBytes();
+            Assert.assertEquals(0, writeOutMedium.getNumLocallyCreated());
+            try {
+              latch.countDown();
+              latch.await();
+              ByteBuffer allocate = ByteBuffer.allocate(4096);
+              writeOutBytes.write(allocate);
+            }
+            catch (Exception e) {
+              throw new RuntimeException(e);
+            }
+            Assert.assertEquals(1, writeOutMedium.getNumLocallyCreated());
+          }
+      );
+    }
+    executorService.awaitTermination(15, TimeUnit.SECONDS);
+    Assert.assertEquals(threadCount, writeOutMedium.getFilesCreated());
+  }
+}
diff --git a/processing/src/test/java/org/apache/druid/segment/writeout/WriteOutBytesTest.java b/processing/src/test/java/org/apache/druid/segment/writeout/WriteOutBytesTest.java
index 9b15bfd0fa2..267707d4870 100644
--- a/processing/src/test/java/org/apache/druid/segment/writeout/WriteOutBytesTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/writeout/WriteOutBytesTest.java
@@ -119,4 +119,14 @@ public class WriteOutBytesTest
     writeOutBytes.write('1');
     writeOutBytes.readFully(1, ByteBuffer.allocate(0));
   }
+
+  @Test
+  public void testWriteWithOffset() throws IOException
+  {
+    WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes();
+    writeOutBytes.write(new byte[] {0x01, 0x02, 0x03, 0x04}, 1, 2);
+    ByteBuffer destination = ByteBuffer.allocate(2);
+    writeOutBytes.readFully(0, destination);
+    Assert.assertArrayEquals(new byte[] {0x02, 0x03}, destination.array());
+  }
 }

From 2abcb41559a6e58e182f8f1ce221507b77b1f55b Mon Sep 17 00:00:00 2001
From: Adarsh Sanjeev <adarshsanjeev@gmail.com>
Date: Fri, 23 Aug 2024 11:37:42 +0530
Subject: [PATCH 48/99] Use controller id while reading from durable storage
 (#16943)

---
 .../main/java/org/apache/druid/msq/exec/RunWorkOrder.java    | 5 ++++-
 .../src/main/java/org/apache/druid/msq/exec/WorkerImpl.java  | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
index 0173979efee..27779f53251 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
@@ -110,6 +110,7 @@ import java.util.stream.Collectors;
  */
 public class RunWorkOrder
 {
+  private final String controllerTaskId;
   private final WorkOrder workOrder;
   private final InputChannelFactory inputChannelFactory;
   private final CounterTracker counterTracker;
@@ -138,6 +139,7 @@ public class RunWorkOrder
   private ListenableFuture<OutputChannels> stageOutputChannelsFuture;
 
   public RunWorkOrder(
+      final String controllerTaskId,
       final WorkOrder workOrder,
       final InputChannelFactory inputChannelFactory,
       final CounterTracker counterTracker,
@@ -150,6 +152,7 @@ public class RunWorkOrder
       final boolean removeNullBytes
   )
   {
+    this.controllerTaskId = controllerTaskId;
     this.workOrder = workOrder;
     this.inputChannelFactory = inputChannelFactory;
     this.counterTracker = counterTracker;
@@ -565,7 +568,7 @@ public class RunWorkOrder
   )
   {
     return DurableStorageOutputChannelFactory.createStandardImplementation(
-        workOrder.getQueryDefinition().getQueryId(),
+        controllerTaskId,
         workOrder.getWorkerNumber(),
         workOrder.getStageNumber(),
         workerContext.workerId(),
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
index 912826c5c5a..90f018d07ae 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
@@ -408,6 +408,7 @@ public class WorkerImpl implements Worker
 
     final QueryContext queryContext = task != null ? QueryContext.of(task.getContext()) : QueryContext.empty();
     final RunWorkOrder runWorkOrder = new RunWorkOrder(
+        task.getControllerTaskId(),
         workOrder,
         inputChannelFactory,
         stageCounters.computeIfAbsent(

From 8c8a4b2302c91f481029fae0e7b1f68ba509b0ba Mon Sep 17 00:00:00 2001
From: AmatyaAvadhanula <amatya.avadhanula@imply.io>
Date: Fri, 23 Aug 2024 13:21:07 +0530
Subject: [PATCH 49/99] Remove references to chatAsync (#16950)

Remove references to chatAsync from Rabbit stream supervisors
---
 .../extensions-contrib/rabbit-stream-ingestion.md        | 2 --
 docs/ingestion/kafka-ingestion.md                        | 2 --
 .../supervisor/RabbitStreamSupervisorTuningConfig.java   | 9 ---------
 .../RabbitStreamIndexTaskTuningConfigTest.java           | 1 -
 .../supervisor/RabbitStreamSupervisorTest.java           | 3 ---
 .../RabbitStreamSupervisorTuningConfigTest.java          | 1 -
 6 files changed, 18 deletions(-)

diff --git a/docs/development/extensions-contrib/rabbit-stream-ingestion.md b/docs/development/extensions-contrib/rabbit-stream-ingestion.md
index 55fedf772ed..9c0e3951807 100644
--- a/docs/development/extensions-contrib/rabbit-stream-ingestion.md
+++ b/docs/development/extensions-contrib/rabbit-stream-ingestion.md
@@ -160,8 +160,6 @@ The `tuningConfig` is optional. If no `tuningConfig` is specified, default param
 |`resetOffsetAutomatically`|Boolean|Controls behavior when Druid needs to read RabbitMQ messages that are no longer available. Not supported.  |no (default == false)|
 |`skipSequenceNumberAvailabilityCheck`|Boolean|Whether to enable checking if the current sequence number is still available in a particular RabbitMQ stream. If set to false, the indexing task will attempt to reset the current sequence number (or not), depending on the value of `resetOffsetAutomatically`.|no (default == false)|
 |`workerThreads`|Integer|The number of threads that the supervisor uses to handle requests/responses for worker tasks, along with any other internal asynchronous operation.|no (default == min(10, taskCount))|
-|`chatAsync`|Boolean| If true, use asynchronous communication with indexing tasks, and ignore the `chatThreads` parameter. If false, use synchronous communication in a thread pool of size `chatThreads`.  | no (default == true) |
-|`chatThreads`|Integer| The number of threads that will be used for communicating with indexing tasks. Ignored if `chatAsync` is `true` (the default).| no (default == min(10, taskCount * replicas))|
 |`chatRetries`|Integer|The number of times HTTP requests to indexing tasks will be retried before considering tasks unresponsive.| no (default == 8)|
 |`httpTimeout`|ISO8601 Period|How long to wait for a HTTP response from an indexing task.|no (default == PT10S)|
 |`shutdownTimeout`|ISO8601 Period|How long to wait for the supervisor to attempt a graceful shutdown of tasks before exiting.|no (default == PT80S)|
diff --git a/docs/ingestion/kafka-ingestion.md b/docs/ingestion/kafka-ingestion.md
index 5a3ac7f29c4..7eb393017d1 100644
--- a/docs/ingestion/kafka-ingestion.md
+++ b/docs/ingestion/kafka-ingestion.md
@@ -421,8 +421,6 @@ For configuration properties shared across all streaming ingestion methods, refe
 |Property|Type|Description|Required|Default|
 |--------|----|-----------|--------|-------|
 |`numPersistThreads`|Integer|The number of threads to use to create and persist incremental segments on the disk. Higher ingestion data throughput results in a larger number of incremental segments, causing significant CPU time to be spent on the creation of the incremental segments on the disk. For datasources with number of columns running into hundreds or thousands, creation of the incremental segments may take up significant time, in the order of multiple seconds. In both of these scenarios, ingestion can stall or pause frequently, causing it to fall behind. You can use additional threads to parallelize the segment creation without blocking ingestion as long as there are sufficient CPU resources available.|No|1|
-|`chatAsync`|Boolean|If `true`, use asynchronous communication with indexing tasks, and ignore the `chatThreads` parameter. If `false`, use synchronous communication in a thread pool of size `chatThreads`.|No|`true`|
-|`chatThreads`|Integer|The number of threads to use for communicating with indexing tasks. Ignored if `chatAsync` is `true`.|No|`min(10, taskCount * replicas)`|
 
 ## Deployment notes on Kafka partitions and Druid segments
 
diff --git a/extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfig.java b/extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfig.java
index 4484cc81199..c6db20eed91 100644
--- a/extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfig.java
+++ b/extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfig.java
@@ -34,8 +34,6 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
     implements SeekableStreamSupervisorTuningConfig
 {
   private final Integer workerThreads;
-  private final Boolean chatAsync;
-  private final Integer chatThreads;
   private final Long chatRetries;
   private final Duration httpTimeout;
   private final Duration shutdownTimeout;
@@ -70,8 +68,6 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
         null,
         null,
         null,
-        null,
-        null,
         null
         );
   }
@@ -92,8 +88,6 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
       @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically,
       @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory,
       @JsonProperty("workerThreads") Integer workerThreads,
-      @JsonProperty("chatAsync") Boolean chatAsync,
-      @JsonProperty("chatThreads") Integer chatThreads,
       @JsonProperty("chatRetries") Long chatRetries,
       @JsonProperty("httpTimeout") Period httpTimeout,
       @JsonProperty("shutdownTimeout") Period shutdownTimeout,
@@ -133,8 +127,6 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
         maxRecordsPerPoll
     );
     this.workerThreads = workerThreads;
-    this.chatAsync = chatAsync;
-    this.chatThreads = chatThreads;
     this.chatRetries = (chatRetries != null ? chatRetries : DEFAULT_CHAT_RETRIES);
     this.httpTimeout = SeekableStreamSupervisorTuningConfig.defaultDuration(httpTimeout, DEFAULT_HTTP_TIMEOUT);
     this.shutdownTimeout = SeekableStreamSupervisorTuningConfig.defaultDuration(
@@ -206,7 +198,6 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
         ", resetOffsetAutomatically=" + isResetOffsetAutomatically() +
         ", segmentWriteOutMediumFactory=" + getSegmentWriteOutMediumFactory() +
         ", workerThreads=" + workerThreads +
-        ", chatThreads=" + chatThreads +
         ", chatRetries=" + chatRetries +
         ", httpTimeout=" + httpTimeout +
         ", shutdownTimeout=" + shutdownTimeout +
diff --git a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java
index 82a57a37b87..852942a8423 100644
--- a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java
+++ b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java
@@ -162,7 +162,6 @@ public class RabbitStreamIndexTaskTuningConfigTest
         "resetOffsetAutomatically=false, " +
         "segmentWriteOutMediumFactory=null, " +
         "workerThreads=null, " +
-        "chatThreads=null, " +
         "chatRetries=8, " +
         "httpTimeout=PT10S, " +
         "shutdownTimeout=PT80S, " +
diff --git a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java
index 44eb77ee463..5f31d70b75d 100644
--- a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java
+++ b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java
@@ -74,7 +74,6 @@ public class RabbitStreamSupervisorTest extends EasyMockSupport
       false,
       false);
   private static final String DATASOURCE = "testDS";
-  private static final int TEST_CHAT_THREADS = 3;
   private static final long TEST_CHAT_RETRIES = 9L;
   private static final Period TEST_HTTP_TIMEOUT = new Period("PT10S");
   private static final Period TEST_SHUTDOWN_TIMEOUT = new Period("PT80S");
@@ -148,8 +147,6 @@ public class RabbitStreamSupervisorTest extends EasyMockSupport
         null,
         null,
         numThreads, // worker threads
-        null,
-        TEST_CHAT_THREADS,
         TEST_CHAT_RETRIES,
         TEST_HTTP_TIMEOUT,
         TEST_SHUTDOWN_TIMEOUT,
diff --git a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfigTest.java b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfigTest.java
index 22f87bcc2b3..63900207829 100644
--- a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfigTest.java
+++ b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfigTest.java
@@ -87,7 +87,6 @@ public class RabbitStreamSupervisorTuningConfigTest
         + "  \"reportParseExceptions\": true,\n"
         + "  \"handoffConditionTimeout\": 100,\n"
         + "  \"workerThreads\": 12,\n"
-        + "  \"chatThreads\": 13,\n"
         + "  \"chatRetries\": 14,\n"
         + "  \"httpTimeout\": \"PT15S\",\n"
         + "  \"shutdownTimeout\": \"PT95S\",\n"

From 0603d5153d8177856687c957ae0061492c81e1d4 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Fri, 23 Aug 2024 08:24:43 -0700
Subject: [PATCH 50/99] Segments sorted by non-time columns. (#16849)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Segments primarily sorted by non-time columns.

Currently, segments are always sorted by __time, followed by the sort
order provided by the user via dimensionsSpec or CLUSTERED BY. Sorting
by __time enables efficient execution of queries involving time-ordering
or granularity. Time-ordering is a simple matter of reading the rows in
stored order, and granular cursors can be generated in streaming fashion.

However, for various workloads, it's better for storage footprint and
query performance to sort by arbitrary orders that do not start with __time.
With this patch, users can sort segments by such orders.

For spec-based ingestion, users add "useExplicitSegmentSortOrder: true" to
dimensionsSpec. The "dimensions" list determines the sort order. To
define a sort order that includes "__time", users explicitly
include a dimension named "__time".

For SQL-based ingestion, users set the context parameter
"useExplicitSegmentSortOrder: true". The CLUSTERED BY clause is then
used as the explicit segment sort order.

In both cases, when the new "useExplicitSegmentSortOrder" parameter is
false (the default), __time is implicitly prepended to the sort order,
as it always was prior to this patch.

The new parameter is experimental for two main reasons. First, such
segments can cause errors when loaded by older servers, due to violating
their expectations that timestamps are always monotonically increasing.
Second, even on newer servers, not all queries can run on non-time-sorted
segments. Scan queries involving time-ordering and any query involving
granularity will not run. (To partially mitigate this, a currently-undocumented
SQL feature "sqlUseGranularity" is provided. When set to false the SQL planner
avoids using "granularity".)

Changes on the write path:

1) DimensionsSpec can now optionally contain a __time dimension, which
   controls the placement of __time in the sort order. If not present,
   __time is considered to be first in the sort order, as it has always
   been.

2) IncrementalIndex and IndexMerger are updated to sort facts more
   flexibly; not always by time first.

3) Metadata (stored in metadata.drd) gains a "sortOrder" field.

4) MSQ can generate range-based shard specs even when not all columns are
   singly-valued strings. It merely stops accepting new clustering key
   fields when it encounters the first one that isn't a singly-valued
   string. This is useful because it enables range shard specs on
   "someDim" to be created for clauses like "CLUSTERED BY someDim, __time".

Changes on the read path:

1) Add StorageAdapter#getSortOrder so query engines can tell how a
   segment is sorted.

2) Update QueryableIndexStorageAdapter, IncrementalIndexStorageAdapter,
   and VectorCursorGranularizer to throw errors when using granularities
   on non-time-ordered segments.

3) Update ScanQueryEngine to throw an error when using the time-ordering
  "order" parameter on non-time-ordered segments.

4) Update TimeBoundaryQueryRunnerFactory to perform a segment scan when
   running on a non-time-ordered segment.

5) Add "sqlUseGranularity" context parameter that causes the SQL planner
   to avoid using granularities other than ALL.

Other changes:

1) Rename DimensionsSpec "hasCustomDimensions" to "hasFixedDimensions"
   and change the meaning subtly: it now returns true if the DimensionsSpec
   represents an unchanging list of dimensions, or false if there is
   some discovery happening. This is what call sites had expected anyway.

* Fixups from CI.

* Fixes.

* Fix missing arg.

* Additional changes.

* Fix logic.

* Fixes.

* Fix test.

* Adjust test.

* Remove throws.

* Fix styles.

* Fix javadocs.

* Cleanup.

* Smoother handling of null ordering.

* Fix tests.

* Missed a spot on the merge.

* Fixups.

* Avoid needless Filters.and.

* Add timeBoundaryInspector to test.

* Fix tests.

* Fix FrameStorageAdapterTest.

* Fix various tests.

* Use forceSegmentSortByTime instead of useExplicitSegmentSortOrder.

* Pom fix.

* Fix doc.
---
 .../ExpressionSelectorBenchmark.java          |   8 +-
 docs/ingestion/ingestion-spec.md              |   2 +-
 docs/ingestion/partitioning.md                |  69 ++-
 docs/multi-stage-query/concepts.md            |  10 +-
 docs/multi-stage-query/reference.md           |  12 +-
 docs/querying/sql-query-context.md            |  10 +-
 .../DistinctCountTimeseriesQueryTest.java     |   8 +-
 .../DistinctCountTopNQueryTest.java           |   8 +-
 .../ApproximateHistogramGroupByQueryTest.java |   2 +-
 .../ApproximateHistogramTopNQueryTest.java    |   3 +-
 ...FixedBucketsHistogramGroupByQueryTest.java |   2 +-
 .../FixedBucketsHistogramTopNQueryTest.java   |   2 +-
 .../apache/druid/msq/exec/ControllerImpl.java | 167 ++++---
 .../GroupByPreShuffleFrameProcessor.java      |   9 +-
 .../druid/msq/sql/MSQTaskQueryMaker.java      |  11 +-
 .../druid/msq/sql/MSQTaskSqlEngine.java       | 113 ++++-
 .../msq/util/MSQTaskQueryMakerUtils.java      |  27 +-
 .../msq/util/MultiStageQueryContext.java      |   9 +
 .../apache/druid/msq/exec/MSQReplaceTest.java | 422 +++++++++++++++++-
 .../msq/exec/TaskDataSegmentProviderTest.java |   7 +
 .../msq/util/MSQTaskQueryMakerUtilsTest.java  |  26 +-
 .../variance/VarianceGroupByQueryTest.java    |  40 +-
 .../variance/VarianceTopNQueryTest.java       |   2 +-
 .../druid/indexer/IndexGeneratorJob.java      |  13 +-
 .../indexer/path/DatasourcePathSpec.java      |   2 +-
 .../common/task/AbstractBatchIndexTask.java   |  17 +-
 .../indexing/common/task/CompactionTask.java  |  75 ++--
 .../druid/indexing/input/InputRowSchemas.java |   2 +-
 .../overlord/sampler/InputSourceSampler.java  |   4 +-
 .../common/task/CompactionTaskRunTest.java    | 159 +++++++
 .../common/task/CompactionTaskTest.java       |   1 +
 .../druid/data/input/impl/DimensionsSpec.java |  87 +++-
 .../frame/segment/FrameStorageAdapter.java    |  19 -
 .../ColumnarFrameCursorHolderFactory.java     |  36 +-
 .../segment/columnar/FrameQueryableIndex.java |   7 +
 .../row/RowFrameCursorHolderFactory.java      |  23 +-
 .../common/granularity/Granularities.java     |   1 -
 .../druid/query/CursorGranularizer.java       |  36 +-
 .../apache/druid/query/FluentQueryRunner.java |  27 --
 .../DataSourceMetadataQueryRunnerFactory.java |  19 +-
 .../groupby/GroupByQueryRunnerFactory.java    |  13 +-
 .../druid/query/groupby/GroupingEngine.java   |  11 +-
 .../epinephelinae/GroupByQueryEngine.java     |  12 +-
 .../vector/VectorGroupByEngine.java           |  39 +-
 .../druid/query/scan/ScanQueryEngine.java     |  17 +-
 .../query/search/UseIndexesStrategy.java      |   5 +-
 .../query/timeboundary/TimeBoundaryQuery.java |  10 +
 .../TimeBoundaryQueryRunnerFactory.java       | 259 ++++++++---
 .../timeseries/TimeseriesQueryEngine.java     |  22 +-
 .../TimeseriesQueryRunnerFactory.java         |  21 +-
 .../druid/query/topn/TopNQueryEngine.java     |  10 +-
 .../query/topn/TopNQueryRunnerFactory.java    |   9 +-
 .../vector/VectorCursorGranularizer.java      |  31 +-
 .../apache/druid/segment/CursorBuildSpec.java |  21 +-
 .../apache/druid/segment/CursorHolder.java    |  26 +-
 .../org/apache/druid/segment/Cursors.java     |  35 +-
 .../druid/segment/FilteredStorageAdapter.java |  19 -
 ...talIndexMaxIngestedEventTimeInspector.java |  39 ++
 .../segment/IncrementalIndexSegment.java      |  15 +
 ...IncrementalIndexTimeBoundaryInspector.java |  57 +++
 .../org/apache/druid/segment/IndexIO.java     |   8 +-
 .../org/apache/druid/segment/IndexMerger.java |  60 ++-
 .../apache/druid/segment/IndexMergerV9.java   |  62 ++-
 .../druid/segment/IndexableAdapter.java       |   8 +-
 .../MaxIngestedEventTimeInspector.java        |  37 ++
 .../org/apache/druid/segment/Metadata.java    |  79 +++-
 .../apache/druid/segment/QueryableIndex.java  |  14 +
 .../QueryableIndexColumnSelectorFactory.java  |  15 +-
 .../segment/QueryableIndexCursorHolder.java   | 248 ++++++----
 .../QueryableIndexIndexableAdapter.java       |  56 ++-
 .../druid/segment/QueryableIndexSegment.java  |  12 +-
 .../segment/QueryableIndexStorageAdapter.java |  40 --
 .../QueryableIndexTimeBoundaryInspector.java  | 100 +++++
 .../druid/segment/RowBasedStorageAdapter.java |  26 +-
 .../RowCombiningTimeAndDimsIterator.java      |   1 +
 .../org/apache/druid/segment/RowPointer.java  |   4 +-
 .../org/apache/druid/segment/Segment.java     |  23 +-
 .../druid/segment/SimpleQueryableIndex.java   |  13 +
 .../SingleScanTimeDimensionSelector.java      |  16 +-
 .../apache/druid/segment/StorageAdapter.java  |  62 ++-
 .../druid/segment/TimeAndDimsPointer.java     |  59 ++-
 .../druid/segment/TimeBoundaryInspector.java  |  62 +++
 .../druid/segment/UnnestStorageAdapter.java   |  39 +-
 .../segment/WrappedSegmentReference.java      |  12 +
 .../segment/WrappedTimeBoundaryInspector.java |  69 +++
 .../segment/incremental/IncrementalIndex.java | 153 +++++--
 .../incremental/IncrementalIndexAdapter.java  |   4 +-
 ...IncrementalIndexColumnSelectorFactory.java |  15 +-
 .../IncrementalIndexCursorHolder.java         |  26 +-
 .../incremental/IncrementalIndexRow.java      |   2 +-
 .../IncrementalIndexRowIterator.java          |   4 +-
 .../IncrementalIndexStorageAdapter.java       |  22 +-
 .../incremental/OnheapIncrementalIndex.java   | 126 +++++-
 .../druid/segment/join/HashJoinEngine.java    |   2 -
 .../druid/segment/join/HashJoinSegment.java   |   7 +-
 .../join/HashJoinSegmentStorageAdapter.java   |  93 ++--
 .../apache/druid/segment/join/Joinable.java   |   2 -
 .../segment/join/lookup/LookupJoinable.java   |   1 -
 .../table/BroadcastSegmentIndexedTable.java   |   5 +-
 .../segment/join/table/IndexedTable.java      |   2 +-
 .../join/table/IndexedTableJoinMatcher.java   |  10 +-
 .../join/table/IndexedTableJoinable.java      |   2 -
 .../loading/TombstoneSegmentizerFactory.java  |   8 +-
 .../segment/virtual/ExpressionSelectors.java  |   6 +-
 .../segment/FrameStorageAdapterTest.java      |  86 ++--
 .../druid/query/CursorGranularizerTest.java   |  20 +-
 .../druid/query/QueryRunnerTestHelper.java    | 173 ++++---
 .../apache/druid/query/TestQueryRunner.java   |  70 +++
 .../first/StringFirstTimeseriesQueryTest.java |  16 +-
 .../last/StringLastTimeseriesQueryTest.java   |  16 +-
 .../groupby/GroupByQueryMergeBufferTest.java  |   2 +-
 .../GroupByQueryRunnerFailureTest.java        |  35 +-
 .../query/groupby/GroupByQueryRunnerTest.java | 362 ++++++++++++++-
 .../GroupByTimeseriesQueryRunnerTest.java     |   2 +-
 .../VectorGroupByEngineIteratorTest.java      |   5 +
 .../query/metadata/SegmentAnalyzerTest.java   |   4 +
 .../SegmentMetadataUnionQueryTest.java        |   8 +-
 .../druid/query/scan/ScanQueryRunnerTest.java |   4 +-
 .../query/search/SearchQueryRunnerTest.java   |   3 +-
 .../TimeBoundaryQueryRunnerTest.java          |   8 +-
 .../TimeSeriesUnionQueryRunnerTest.java       |   3 +-
 .../timeseries/TimeseriesQueryRunnerTest.java |   3 +-
 .../topn/TopNMetricSpecOptimizationsTest.java |  21 -
 .../druid/query/topn/TopNQueryRunnerTest.java |  46 +-
 .../druid/query/topn/TopNUnionQueryTest.java  |   2 +-
 .../IndexMergerLongestSharedDimOrderTest.java |   6 +-
 .../druid/segment/IndexMergerTestBase.java    | 364 ++++++++++++++-
 .../druid/segment/MergingRowIteratorTest.java |   2 +
 .../apache/druid/segment/MetadataTest.java    |  98 +++-
 .../segment/RowBasedStorageAdapterTest.java   |  44 +-
 .../org/apache/druid/segment/TestIndex.java   |  74 ++-
 .../TombstoneSegmentStorageAdapterTest.java   |  20 -
 .../segment/UnnestStorageAdapterTest.java     |  13 -
 .../segment/data/IncrementalIndexTest.java    |   9 +-
 .../IncrementalIndexRowCompTest.java          |  92 +++-
 .../IncrementalIndexStorageAdapterTest.java   | 108 +++--
 .../HashJoinSegmentStorageAdapterTest.java    |  28 --
 .../segment/join/HashJoinSegmentTest.java     |  29 ++
 .../BroadcastSegmentIndexedTableTest.java     |   2 +-
 .../join/table/IndexedTableJoinableTest.java  |   1 -
 .../DimensionRangeShardSpecTest.java          |  23 +
 .../druid/segment/indexing/DataSchema.java    |  48 +-
 .../appenderator/SinkQuerySegmentWalker.java  |  19 +-
 .../druid/segment/realtime/sink/Sink.java     |   8 +-
 .../server/coordination/ServerManager.java    |  10 +-
 .../segment/indexing/DataSchemaTest.java      | 125 +++++-
 .../apache/druid/server/TestSegmentUtils.java |  28 +-
 .../org/apache/druid/cli/DumpSegmentTest.java |   4 +-
 .../sql/calcite/planner/PlannerContext.java   |  29 ++
 .../druid/sql/calcite/rel/DruidQuery.java     |   7 +-
 .../sql/calcite/CalciteJoinQueryTest.java     |   6 +-
 .../druid/sql/calcite/CalciteQueryTest.java   | 113 +++++
 .../calcite/CalciteTimeBoundaryQueryTest.java |   3 +
 153 files changed, 4615 insertions(+), 1370 deletions(-)
 create mode 100644 processing/src/main/java/org/apache/druid/segment/IncrementalIndexMaxIngestedEventTimeInspector.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/IncrementalIndexTimeBoundaryInspector.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/MaxIngestedEventTimeInspector.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/QueryableIndexTimeBoundaryInspector.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/TimeBoundaryInspector.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/WrappedTimeBoundaryInspector.java
 create mode 100644 processing/src/test/java/org/apache/druid/query/TestQueryRunner.java

diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java
index 5bc67bc9f72..8d79a611d0e 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java
@@ -39,9 +39,11 @@ import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexTimeBoundaryInspector;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
@@ -192,11 +194,11 @@ public class ExpressionSelectorBenchmark
     try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
       final Cursor cursor = cursorHolder.asCursor();
       final CursorGranularizer granularizer = CursorGranularizer.create(
-          adapter,
           cursor,
+          QueryableIndexTimeBoundaryInspector.create(index),
+          Cursors.getTimeOrdering(index.getOrdering()),
           Granularities.HOUR,
-          adapter.getInterval(),
-          false
+          adapter.getInterval()
       );
       final Sequence<Long> results =
           Sequences.simple(granularizer.getBucketIterable())
diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md
index a1a0ff1ee5b..37f0bf0a1fd 100644
--- a/docs/ingestion/ingestion-spec.md
+++ b/docs/ingestion/ingestion-spec.md
@@ -232,7 +232,7 @@ A `dimensionsSpec` can have the following components:
 | `spatialDimensions`    | An array of [spatial dimensions](../querying/geo.md).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | `[]`    |
 | `includeAllDimensions` | Note that this field only applies to string-based schema discovery where Druid ingests dimensions it discovers as strings. This is different from schema auto-discovery where Druid infers the type for data. You can set `includeAllDimensions` to true to ingest both explicit dimensions in the `dimensions` field and other dimensions that the ingestion task discovers from input data. In this case, the explicit dimensions will appear first in the order that you specify them, and the dimensions dynamically discovered will come after. This flag can be useful especially with auto schema discovery using [`flattenSpec`](./data-formats.md#flattenspec). If this is not set and the `dimensions` field is not empty, Druid will ingest only explicit dimensions. If this is not set and the `dimensions` field is empty, all discovered dimensions will be ingested. | false   |
 | `useSchemaDiscovery` | Configure Druid to use schema auto-discovery to discover some or all of the dimensions and types for your data. For any dimensions that aren't a uniform type, Druid ingests them as JSON. You can use this for native batch or streaming ingestion.  | false  | 
-
+| `forceSegmentSortByTime` | When set to true (the default), segments created by the ingestion job are sorted by `{__time, dimensions[0], dimensions[1], ...}`. When set to false, segments created by the ingestion job are sorted by `{dimensions[0], dimensions[1], ...}`. To include `__time` in the sort order when this parameter is set to `false`, you must include a dimension named `__time` with type `long` explicitly in the `dimensions` list.<br /><br />Setting this to `false` is an experimental feature; see [Sorting](partitioning.md#sorting) for details. | `true` |
 
 #### Dimension objects
 
diff --git a/docs/ingestion/partitioning.md b/docs/ingestion/partitioning.md
index 6cf5b0a74d2..d1501f5fe0f 100644
--- a/docs/ingestion/partitioning.md
+++ b/docs/ingestion/partitioning.md
@@ -44,33 +44,60 @@ Partitioning by time is important for two reasons:
 The most common choices to balance these considerations are `hour` and `day`. For streaming ingestion, `hour` is especially
 common, because it allows compaction to follow ingestion with less of a time delay.
 
+The following table describes how to configure time chunk partitioning.
+
+|Method|Configuration|
+|------|------------|
+|[SQL](../multi-stage-query/index.md)|[`PARTITIONED BY`](../multi-stage-query/concepts.md#partitioning)|
+|[Kafka](../ingestion/kafka-ingestion.md) or [Kinesis](../ingestion/kinesis-ingestion.md)|`segmentGranularity` inside the [`granularitySpec`](ingestion-spec.md#granularityspec)|
+|[Native batch](native-batch.md) or [Hadoop](hadoop.md)|`segmentGranularity` inside the [`granularitySpec`](ingestion-spec.md#granularityspec)|
+
 ## Secondary partitioning
 
-Druid can partition segments within a particular time chunk further depending upon options that vary based on the ingestion type you have chosen. In general, secondary partitioning on a particular dimension improves locality. This means that rows with the same value for that dimension are stored together, decreasing access time.
+Druid further partitions each time chunk into immutable segments. Secondary partitioning on a particular dimension improves locality. This means that rows with the same value for that dimension are stored together, decreasing access time.
 
-To achieve the best performance and smallest overall footprint, partition your data on a "natural"
-dimension that you often use as a filter when possible. Such partitioning often improves compression and query performance. For example, some cases have yielded threefold storage size decreases.
+To achieve the best performance and smallest overall footprint, partition your data on a "natural" dimension that
+you often use as a filter, or that achieves some alignment within your data. Such partitioning can improve compression
+and query performance by significant multiples.
 
-## Partitioning and sorting
+The following table describes how to configure secondary partitioning.
 
-Partitioning and sorting work well together. If you do have a "natural" partitioning dimension, consider placing it first in the `dimensions` list of your `dimensionsSpec`. This way Druid sorts rows within each segment by that column. This sorting configuration frequently improves compression more than using partitioning alone.
-
-Note that Druid always sorts rows within a segment by timestamp first, even before the first dimension listed in your `dimensionsSpec`. This sorting can preclude the efficacy of dimension sorting. To work around this limitation if necessary, set your `queryGranularity` equal to `segmentGranularity` in your [`granularitySpec`](./ingestion-spec.md#granularityspec). Druid will set all timestamps within the segment to the same value, letting you identify a [secondary timestamp](schema-design.md#secondary-timestamps) as the "real" timestamp.
-
-## How to configure partitioning
-
-Not all ingestion methods support an explicit partitioning configuration, and not all have equivalent levels of flexibility. If you are doing initial ingestion through a less-flexible method like
-Kafka), you can use [reindexing](../data-management/update.md#reindex) or [compaction](../data-management/compaction.md) to repartition your data after initial ingestion. This is a powerful technique you can use to optimally partition any data older than a certain time threshold while you continuously add new data from a stream.
-
-The following table shows how each ingestion method handles partitioning:
-
-|Method|How it works|
+|Method|Configuration|
 |------|------------|
-|[Native batch](native-batch.md)|Configured using [`partitionsSpec`](native-batch.md#partitionsspec) inside the `tuningConfig`.|
-|[SQL](../multi-stage-query/index.md)|Configured using [`PARTITIONED BY`](../multi-stage-query/concepts.md#partitioning) and [`CLUSTERED BY`](../multi-stage-query/concepts.md#clustering).|
-|[Hadoop](hadoop.md)|Configured using [`partitionsSpec`](hadoop.md#partitionsspec) inside the `tuningConfig`.|
-|[Kafka indexing service](../ingestion/kafka-ingestion.md)|Kafka topic partitioning defines how Druid partitions the datasource. You can also [reindex](../data-management/update.md#reindex) or [compact](../data-management/compaction.md) to repartition after initial ingestion.|
-|[Kinesis indexing service](../ingestion/kinesis-ingestion.md)|Kinesis stream sharding defines how Druid partitions the datasource. You can also [reindex](../data-management/update.md#reindex) or [compact](../data-management/compaction.md) to repartition after initial ingestion.|
+|[SQL](../multi-stage-query/index.md)|[`CLUSTERED BY`](../multi-stage-query/concepts.md#clustering)|
+|[Kafka](../ingestion/kafka-ingestion.md) or [Kinesis](../ingestion/kinesis-ingestion.md)|Upstream partitioning defines how Druid partitions the datasource. You can also alter clustering using [`REPLACE`](../multi-stage-query/concepts.md#replace) (with `CLUSTERED BY`) or [compaction](../data-management/compaction.md) after initial ingestion.|
+|[Native batch](native-batch.md) or [Hadoop](hadoop.md)|[`partitionsSpec`](native-batch.md#partitionsspec) inside the `tuningConfig`|
+
+## Sorting
+
+Each segment is internally sorted to promote compression and locality.
+
+Partitioning and sorting work well together. If you do have a "natural" partitioning dimension, consider placing it
+first in your sort order as well. This way, Druid sorts rows within each segment by that column. This sorting configuration
+frequently improves compression and performance more than using partitioning alone.
+
+The following table describes how to configure sorting.
+
+|Method|Configuration|
+|------|------------|
+|[SQL](../multi-stage-query/index.md)|Uses order of fields in [`CLUSTERED BY`](../multi-stage-query/concepts.md#clustering) or [`segmentSortOrder`](../multi-stage-query/reference.md#context) in the query context|
+|[Kafka](../ingestion/kafka-ingestion.md) or [Kinesis](../ingestion/kinesis-ingestion.md)|Uses order of fields in [`dimensionsSpec`](ingestion-spec.md#granularityspec)|
+|[Native batch](native-batch.md) or [Hadoop](hadoop.md)|Uses order of fields in [`dimensionsSpec`](ingestion-spec.md#granularityspec)|
+
+:::info
+Druid implicitly sorts rows within a segment by `__time` first before any `dimensions` or `CLUSTERED BY` fields, unless
+you set `forceSegmentSortByTime` to `false` in your
+[query context](../multi-stage-query/reference.md#context-parameters) (for SQL) or in your
+[`dimensionsSpec`](ingestion-spec.md#dimensionsspec) (for other ingestion forms).
+
+Setting `forceSegmentSortByTime` to `false` is an experimental feature. Segments created with sort orders that
+do not start with `__time` can only be read by Druid 31 or later. Additionally, at this time, certain queries are not
+supported on such segments, including:
+
+- Native queries with `granularity` other than `all`.
+- Native `scan` query with ascending or descending time order.
+- SQL queries that plan into an unsupported native query.
+:::
 
 ## Learn more
 
diff --git a/docs/multi-stage-query/concepts.md b/docs/multi-stage-query/concepts.md
index cae88a0f375..27cde86e9a5 100644
--- a/docs/multi-stage-query/concepts.md
+++ b/docs/multi-stage-query/concepts.md
@@ -172,13 +172,13 @@ Clustering is important for two reasons:
 
 1. Lower storage footprint due to improved locality, and therefore improved compressibility.
 2. Better query performance due to dimension-based segment pruning, which removes segments from consideration when they
-   cannot possibly contain data matching a query's filter. This speeds up filters like `x = 'foo'` and `x IN ('foo',
-   'bar')`.
+   cannot possibly contain data matching a query's filter. This speeds up filters like `x = 'foo'` and
+   `x IN ('foo', 'bar')`.
 
 To activate dimension-based pruning, these requirements must be met:
 
 - Segments were generated by a `REPLACE` statement, not an `INSERT` statement.
-- All `CLUSTERED BY` columns are single-valued string columns.
+- `CLUSTERED BY` begins with single-valued string columns. These single-valued string columns are used for pruning.
 
 If these requirements are _not_ met, Druid still clusters data during ingestion but will not be able to perform
 dimension-based segment pruning at query time. You can tell if dimension-based segment pruning is possible by using the
@@ -188,6 +188,10 @@ available in the **Segments** view under the **Partitioning** column.
 
 For more information about syntax, see [`CLUSTERED BY`](./reference.md#clustered-by).
 
+For more information about the mechanics of clustering, refer to
+[Secondary partitioning](../ingestion/partitioning#secondary-partitioning) and
+[Sorting](../ingestion/partitioning#sorting).
+
 ### Rollup
 
 [Rollup](../ingestion/rollup.md) is a technique that pre-aggregates data during ingestion to reduce the amount of data
diff --git a/docs/multi-stage-query/reference.md b/docs/multi-stage-query/reference.md
index 0f8e710a59f..c4345fedf81 100644
--- a/docs/multi-stage-query/reference.md
+++ b/docs/multi-stage-query/reference.md
@@ -362,8 +362,13 @@ For more information about partitioning, see [Partitioning](concepts.md#partitio
 ### `CLUSTERED BY`
 
 The `CLUSTERED BY <column list>` clause is optional for [INSERT](#insert) and [REPLACE](#replace). It accepts a list of
-column names or expressions. Druid's segment generation only supports ascending order, so an `INSERT` or `REPLACE` query with
-`CLUSTERED BY` columns in `DESC` ordering is not allowed.
+column names or expressions.
+
+This column list is used for [secondary partitioning](../ingestion/partitioning.md#secondary-partitioning) of segments
+within a time chunk, and [sorting](../ingestion/partitioning.md#sorting) of rows within a segment. For sorting purposes,
+Druid implicitly prepends `__time` to the `CLUSTERED BY` column list, unless
+[`forceSegmentSortByTime`](#context) is set to `false`
+(an experimental feature; see [Sorting](../ingestion/partitioning.md#sorting) for details).
 
 For more information about clustering, see [Clustering](concepts.md#clustering).
 
@@ -397,7 +402,8 @@ The following table lists the context parameters for the MSQ task engine:
 | `arrayIngestMode` | INSERT, REPLACE<br /><br /> Controls how ARRAY type values are stored in Druid segments. When set to `array` (recommended for SQL compliance), Druid will store all ARRAY typed values in [ARRAY typed columns](../querying/arrays.md), and supports storing both VARCHAR and numeric typed arrays. When set to `mvd` (the default, for backwards compatibility), Druid only supports VARCHAR typed arrays, and will store them as [multi-value string columns](../querying/multi-value-dimensions.md). See [`arrayIngestMode`] in the [Arrays](../querying/arrays.md) page for more details. | `mvd` (for backwards compatibility, recommended to use `array` for SQL compliance)|
 | `sqlJoinAlgorithm` | SELECT, INSERT, REPLACE<br /><br />Algorithm to use for JOIN. Use `broadcast` (the default) for broadcast hash join or `sortMerge` for sort-merge join. Affects all JOIN operations in the query. This is a hint to the MSQ engine and the actual joins in the query may proceed in a different way than specified. See [Joins](#joins) for more details. | `broadcast` |
 | `rowsInMemory` | INSERT or REPLACE<br /><br />Maximum number of rows to store in memory at once before flushing to disk during the segment generation process. Ignored for non-INSERT queries. In most cases, use the default value. You may need to override the default if you run into one of the [known issues](./known-issues.md) around memory usage. | 100,000 |
-| `segmentSortOrder` | INSERT or REPLACE<br /><br />Normally, Druid sorts rows in individual segments using `__time` first, followed by the [CLUSTERED BY](#clustered-by) clause. When you set `segmentSortOrder`, Druid sorts rows in segments using this column list first, followed by the CLUSTERED BY order.<br /><br />You provide the column list as comma-separated values or as a JSON array in string form. If your query includes `__time`, then this list must begin with `__time`. For example, consider an INSERT query that uses `CLUSTERED BY country` and has `segmentSortOrder` set to `__time,city`. Within each time chunk, Druid assigns rows to segments based on `country`, and then within each of those segments, Druid sorts those rows by `__time` first, then `city`, then `country`. | empty list |
+| `segmentSortOrder` | INSERT or REPLACE<br /><br />Normally, Druid sorts rows in individual segments using `__time` first, followed by the [CLUSTERED BY](#clustered-by) clause. When you set `segmentSortOrder`, Druid uses the order from this context parameter instead. Provide the column list as comma-separated values or as a JSON array in string form.<br />< br/>For example, consider an INSERT query that uses `CLUSTERED BY country` and has `segmentSortOrder` set to `__time,city,country`. Within each time chunk, Druid assigns rows to segments based on `country`, and then within each of those segments, Druid sorts those rows by `__time` first, then `city`, then `country`. | empty list |
+| `forceSegmentSortByTime` | INSERT or REPLACE<br /><br />When set to `true` (the default), Druid prepends `__time` to [CLUSTERED BY](#clustered-by) when determining the sort order for individual segments. Druid also requires that `segmentSortOrder`, if provided, starts with `__time`.<br /><br />When set to `false`, Druid uses the [CLUSTERED BY](#clustered-by) alone to determine the sort order for individual segments, and does not require that `segmentSortOrder` begin with `__time`. Setting this parameter to `false` is an experimental feature; see [Sorting](../ingestion/partitioning#sorting) for details. | `true` |
 | `maxParseExceptions`| SELECT, INSERT, REPLACE<br /><br />Maximum number of parse exceptions that are ignored while executing the query before it stops with `TooManyWarningsFault`. To ignore all the parse exceptions, set the value to -1. | 0 |
 | `rowsPerSegment` | INSERT or REPLACE<br /><br />The number of rows per segment to target. The actual number of rows per segment may be somewhat higher or lower than this number. In most cases, use the default. For general information about sizing rows per segment, see [Segment Size Optimization](../operations/segment-optimization.md). | 3,000,000 |
 | `indexSpec` | INSERT or REPLACE<br /><br />An [`indexSpec`](../ingestion/ingestion-spec.md#indexspec) to use when generating segments. May be a JSON string or object. See [Front coding](../ingestion/ingestion-spec.md#front-coding) for details on configuring an `indexSpec` with front coding. | See [`indexSpec`](../ingestion/ingestion-spec.md#indexspec). |
diff --git a/docs/querying/sql-query-context.md b/docs/querying/sql-query-context.md
index dc73c9e1ab3..e003ceec69d 100644
--- a/docs/querying/sql-query-context.md
+++ b/docs/querying/sql-query-context.md
@@ -50,11 +50,11 @@ Configure Druid SQL query planning using the parameters in the table below.
 |`sqlReverseLookup`|Whether to consider the [reverse-lookup rewrite](lookups.md#reverse-lookup) of the `LOOKUP` function during SQL planning.<br /><br />Calls to `LOOKUP` are only reversed when the number of matching keys is lower than both `inSubQueryThreshold` and `sqlReverseLookupThreshold`.|true|
 |`sqlReverseLookupThreshold`|Maximum size of `IN` filter to create when applying a [reverse-lookup rewrite](lookups.md#reverse-lookup). If a `LOOKUP` call matches more keys than this threshold, it is left as-is.<br /><br />If `inSubQueryThreshold` is lower than `sqlReverseLookupThreshold`, the `inSubQueryThreshold` is used as the threshold instead.|10000|
 |`sqlPullUpLookup`|Whether to consider the [pull-up rewrite](lookups.md#pull-up) of the `LOOKUP` function during SQL planning.|true|
-|`enableJoinLeftTableScanDirect`|`false`|This flag applies to queries which have joins. For joins, where left child is a simple scan with a filter,  by default, druid will run the scan as a query and the join the results to the right child on broker. Setting this flag to true overrides that behavior and druid will attempt to push the join to data servers instead. Please note that the flag could be applicable to queries even if there is no explicit join. since queries can internally translated into a join by the SQL planner.|
-|`maxNumericInFilters`|`-1`|Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates only to an [OR](../querying/filters.md#or) of [Bound filter](../querying/filters.md#bound-filter). By default, Druid does not restrict the amount of of numeric Bound Filters on String columns, although this situation may block other queries from running. Set this parameter to a smaller value to prevent Druid from running queries that have prohibitively long segment processing times. The optimal limit requires some trial and error; we recommend starting with 100.  Users who submit a query that exceeds the limit of `maxNumericInFilters` should instead rewrite their queries to use strings in the `WHERE` clause instead of numbers. For example, `WHERE someString IN (‘123’, ‘456’)`. This value cannot exceed the set system configuration `druid.sql.planner.maxNumericInFilters`. This value is ignored if `druid.sql.planner.maxNumericInFilters` is not set explicitly.|
-|`inFunctionThreshold`|`100`| At or beyond this threshold number of values, SQL `IN` is converted to [`SCALAR_IN_ARRAY`](sql-functions.md#scalar_in_array). A threshold of 0 forces this conversion in all cases. A threshold of [Integer.MAX_VALUE] disables this conversion. The converted function is eligible for fewer planning-time optimizations, which speeds up planning, but may prevent certain planning-time optimizations.|
-|`inFunctionExprThreshold`|`2`| At or beyond this threshold number of values, SQL `IN` is eligible for execution using the native function `scalar_in_array` rather than an <code>&#124;&#124;</code> of `==`, even if the number of values is below `inFunctionThreshold`. This property only affects translation of SQL `IN` to a [native expression](math-expr.md). It does not affect translation of SQL `IN` to a [native filter](filters.md). This property is provided for backwards compatibility purposes, and may be removed in a future release.|
-|`inSubQueryThreshold`|`2147483647`| At or beyond this threshold number of values, SQL `IN` is converted to `JOIN` on an inline table. `inFunctionThreshold` takes priority over this setting. A threshold of 0 forces usage of an inline table in all cases where the size of a SQL `IN` is larger than `inFunctionThreshold`. A threshold of `2147483647` disables the rewrite of SQL `IN` to `JOIN`. |
+|`enableJoinLeftTableScanDirect`|This flag applies to queries which have joins. For joins, where left child is a simple scan with a filter,  by default, druid will run the scan as a query and the join the results to the right child on broker. Setting this flag to true overrides that behavior and druid will attempt to push the join to data servers instead. Please note that the flag could be applicable to queries even if there is no explicit join. since queries can internally translated into a join by the SQL planner.|`false`|
+|`maxNumericInFilters`|Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates only to an [OR](../querying/filters.md#or) of [Bound filter](../querying/filters.md#bound-filter). By default, Druid does not restrict the amount of of numeric Bound Filters on String columns, although this situation may block other queries from running. Set this parameter to a smaller value to prevent Druid from running queries that have prohibitively long segment processing times. The optimal limit requires some trial and error; we recommend starting with 100.  Users who submit a query that exceeds the limit of `maxNumericInFilters` should instead rewrite their queries to use strings in the `WHERE` clause instead of numbers. For example, `WHERE someString IN (‘123’, ‘456’)`. This value cannot exceed the set system configuration `druid.sql.planner.maxNumericInFilters`. This value is ignored if `druid.sql.planner.maxNumericInFilters` is not set explicitly.|`-1`|
+|`inFunctionThreshold`| At or beyond this threshold number of values, SQL `IN` is converted to [`SCALAR_IN_ARRAY`](sql-functions.md#scalar_in_array). A threshold of 0 forces this conversion in all cases. A threshold of [Integer.MAX_VALUE] disables this conversion. The converted function is eligible for fewer planning-time optimizations, which speeds up planning, but may prevent certain planning-time optimizations.| `100`|
+|`inFunctionExprThreshold`|At or beyond this threshold number of values, SQL `IN` is eligible for execution using the native function `scalar_in_array` rather than an <code>&#124;&#124;</code> of `==`, even if the number of values is below `inFunctionThreshold`. This property only affects translation of SQL `IN` to a [native expression](math-expr.md). It does not affect translation of SQL `IN` to a [native filter](filters.md). This property is provided for backwards compatibility purposes, and may be removed in a future release.|`2`|
+|`inSubQueryThreshold`|At or beyond this threshold number of values, SQL `IN` is converted to `JOIN` on an inline table. `inFunctionThreshold` takes priority over this setting. A threshold of 0 forces usage of an inline table in all cases where the size of a SQL `IN` is larger than `inFunctionThreshold`. A threshold of `2147483647` disables the rewrite of SQL `IN` to `JOIN`. |`2147483647`|
 
 ## Setting the query context
 The query context parameters can be specified as a "context" object in the [JSON API](../api-reference/sql-api.md) or as a [JDBC connection properties object](../api-reference/sql-jdbc.md).
diff --git a/extensions-contrib/distinctcount/src/test/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountTimeseriesQueryTest.java b/extensions-contrib/distinctcount/src/test/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountTimeseriesQueryTest.java
index c5faefc65ca..c06b5649681 100644
--- a/extensions-contrib/distinctcount/src/test/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountTimeseriesQueryTest.java
+++ b/extensions-contrib/distinctcount/src/test/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountTimeseriesQueryTest.java
@@ -32,6 +32,7 @@ import org.apache.druid.query.timeseries.DefaultTimeseriesQueryMetrics;
 import org.apache.druid.query.timeseries.TimeseriesQuery;
 import org.apache.druid.query.timeseries.TimeseriesQueryEngine;
 import org.apache.druid.query.timeseries.TimeseriesResultValue;
+import org.apache.druid.segment.IncrementalIndexTimeBoundaryInspector;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.incremental.IncrementalIndex;
 import org.apache.druid.segment.incremental.IncrementalIndexSchema;
@@ -101,7 +102,12 @@ public class DistinctCountTimeseriesQueryTest extends InitializedNullHandlingTes
                                   .build();
 
     final Iterable<Result<TimeseriesResultValue>> results =
-        engine.process(query, new IncrementalIndexStorageAdapter(index), new DefaultTimeseriesQueryMetrics()).toList();
+        engine.process(
+            query,
+            new IncrementalIndexStorageAdapter(index),
+            new IncrementalIndexTimeBoundaryInspector(index),
+            new DefaultTimeseriesQueryMetrics()
+        ).toList();
 
     List<Result<TimeseriesResultValue>> expectedResults = Collections.singletonList(
         new Result<>(
diff --git a/extensions-contrib/distinctcount/src/test/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountTopNQueryTest.java b/extensions-contrib/distinctcount/src/test/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountTopNQueryTest.java
index c61a793ff83..ff5a0c31363 100644
--- a/extensions-contrib/distinctcount/src/test/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountTopNQueryTest.java
+++ b/extensions-contrib/distinctcount/src/test/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountTopNQueryTest.java
@@ -33,6 +33,7 @@ import org.apache.druid.query.topn.TopNQuery;
 import org.apache.druid.query.topn.TopNQueryBuilder;
 import org.apache.druid.query.topn.TopNQueryEngine;
 import org.apache.druid.query.topn.TopNResultValue;
+import org.apache.druid.segment.IncrementalIndexTimeBoundaryInspector;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.incremental.IncrementalIndex;
 import org.apache.druid.segment.incremental.IncrementalIndexSchema;
@@ -130,7 +131,12 @@ public class DistinctCountTopNQueryTest extends InitializedNullHandlingTest
                           .build();
 
     final Iterable<Result<TopNResultValue>> results =
-        engine.query(query, new IncrementalIndexStorageAdapter(index), null).toList();
+        engine.query(
+            query,
+            new IncrementalIndexStorageAdapter(index),
+            new IncrementalIndexTimeBoundaryInspector(index),
+            null
+        ).toList();
 
     List<Result<TopNResultValue>> expectedResults = Collections.singletonList(
         new Result<>(
diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java
index 5be792966a9..ca4642bde2f 100644
--- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java
+++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java
@@ -97,7 +97,7 @@ public class ApproximateHistogramGroupByQueryTest extends InitializedNullHandlin
 
     for (GroupByQueryConfig config : configs) {
       final GroupByQueryRunnerFactory factory = GroupByQueryRunnerTest.makeQueryRunnerFactory(config, BUFFER_POOLS);
-      for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunnersToMerge(factory)) {
+      for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunnersToMerge(factory, false)) {
         final String testName = StringUtils.format(
             "config=%s, runner=%s",
             config.toString(),
diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java
index 410e6bd3c5d..ae6c634ad0b 100644
--- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java
+++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java
@@ -61,7 +61,8 @@ public class ApproximateHistogramTopNQueryTest extends InitializedNullHandlingTe
   @Parameterized.Parameters(name = "{0}")
   public static Iterable<Object[]> constructorFeeder()
   {
-    return QueryRunnerTestHelper.transformToConstructionFeeder(TopNQueryRunnerTest.queryRunners());
+    // Don't test with non-time-ordered segments, because results vary too much due to order-dependence.
+    return QueryRunnerTestHelper.transformToConstructionFeeder(TopNQueryRunnerTest.queryRunners(false));
   }
 
   private final QueryRunner runner;
diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramGroupByQueryTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramGroupByQueryTest.java
index 49f91ada006..983ffc00630 100644
--- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramGroupByQueryTest.java
+++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramGroupByQueryTest.java
@@ -97,7 +97,7 @@ public class FixedBucketsHistogramGroupByQueryTest extends InitializedNullHandli
 
     for (GroupByQueryConfig config : configs) {
       final GroupByQueryRunnerFactory factory = GroupByQueryRunnerTest.makeQueryRunnerFactory(config, BUFFER_POOLS);
-      for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunnersToMerge(factory)) {
+      for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunnersToMerge(factory, true)) {
         final String testName = StringUtils.format(
             "config=%s, runner=%s",
             config.toString(),
diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTopNQueryTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTopNQueryTest.java
index 7ea3dd31e59..cead791167d 100644
--- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTopNQueryTest.java
+++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTopNQueryTest.java
@@ -61,7 +61,7 @@ public class FixedBucketsHistogramTopNQueryTest extends InitializedNullHandlingT
   @Parameterized.Parameters(name = "{0}")
   public static Iterable<Object[]> constructorFeeder()
   {
-    return QueryRunnerTestHelper.transformToConstructionFeeder(TopNQueryRunnerTest.queryRunners());
+    return QueryRunnerTestHelper.transformToConstructionFeeder(TopNQueryRunnerTest.queryRunners(true));
   }
 
   private final QueryRunner runner;
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
index 4310619f81f..875212605b8 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
@@ -40,6 +40,7 @@ import org.apache.druid.common.guava.FutureUtils;
 import org.apache.druid.data.input.StringTuple;
 import org.apache.druid.data.input.impl.DimensionSchema;
 import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.LongDimensionSchema;
 import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.discovery.BrokerClient;
 import org.apache.druid.error.DruidException;
@@ -1095,24 +1096,20 @@ public class ControllerImpl implements Controller
     final RowKeyReader keyReader = clusterBy.keyReader(signature);
     final SegmentIdWithShardSpec[] retVal = new SegmentIdWithShardSpec[partitionBoundaries.size()];
     final Granularity segmentGranularity = destination.getSegmentGranularity();
-    final List<String> shardColumns;
-    final Pair<List<String>, String> shardReasonPair;
-
-    shardReasonPair = computeShardColumns(
+    final Pair<List<String>, String> shardReasonPair = computeShardColumns(
         signature,
         clusterBy,
         querySpec.getColumnMappings(),
         mayHaveMultiValuedClusterByFields
     );
+    final List<String> shardColumns = shardReasonPair.lhs;
+    final String commentary = shardReasonPair.rhs;
 
-    shardColumns = shardReasonPair.lhs;
-    String reason = shardReasonPair.rhs;
-    log.info("ShardSpec chosen: %s", reason);
-
+    log.info("ShardSpec chosen: %s", commentary);
     if (shardColumns.isEmpty()) {
-      segmentReport = new MSQSegmentReport(NumberedShardSpec.class.getSimpleName(), reason);
+      segmentReport = new MSQSegmentReport(NumberedShardSpec.class.getSimpleName(), commentary);
     } else {
-      segmentReport = new MSQSegmentReport(DimensionRangeShardSpec.class.getSimpleName(), reason);
+      segmentReport = new MSQSegmentReport(DimensionRangeShardSpec.class.getSimpleName(), commentary);
     }
 
     // Group partition ranges by bucket (time chunk), so we can generate shardSpecs for each bucket independently.
@@ -1156,9 +1153,13 @@ public class ControllerImpl implements Controller
         } else {
           final ClusterByPartition range = ranges.get(segmentNumber).rhs;
           final StringTuple start =
-              segmentNumber == 0 ? null : makeStringTuple(clusterBy, keyReader, range.getStart());
+              segmentNumber == 0
+              ? null
+              : makeStringTuple(clusterBy, keyReader, range.getStart(), shardColumns.size());
           final StringTuple end =
-              segmentNumber == ranges.size() - 1 ? null : makeStringTuple(clusterBy, keyReader, range.getEnd());
+              segmentNumber == ranges.size() - 1
+              ? null
+              : makeStringTuple(clusterBy, keyReader, range.getEnd(), shardColumns.size());
 
           shardSpec = new DimensionRangeShardSpec(shardColumns, start, end, segmentNumber, ranges.size());
         }
@@ -1942,12 +1943,15 @@ public class ControllerImpl implements Controller
   {
     final DataSourceMSQDestination destination = (DataSourceMSQDestination) querySpec.getDestination();
     final boolean isRollupQuery = isRollupQuery(querySpec.getQuery());
+    final boolean forceSegmentSortByTime =
+        MultiStageQueryContext.isForceSegmentSortByTime(querySpec.getQuery().context());
 
-    final Pair<List<DimensionSchema>, List<AggregatorFactory>> dimensionsAndAggregators =
+    final Pair<DimensionsSpec, List<AggregatorFactory>> dimensionsAndAggregators =
         makeDimensionsAndAggregatorsForIngestion(
             querySignature,
             queryClusterBy,
             destination.getSegmentSortOrder(),
+            forceSegmentSortByTime,
             columnMappings,
             isRollupQuery,
             querySpec.getQuery(),
@@ -1957,7 +1961,7 @@ public class ControllerImpl implements Controller
     return new DataSchema(
         destination.getDataSource(),
         new TimestampSpec(ColumnHolder.TIME_COLUMN_NAME, "millis", null),
-        new DimensionsSpec(dimensionsAndAggregators.lhs),
+        dimensionsAndAggregators.lhs,
         dimensionsAndAggregators.rhs.toArray(new AggregatorFactory[0]),
         makeGranularitySpecForIngestion(querySpec.getQuery(), querySpec.getColumnMappings(), isRollupQuery, jsonMapper),
         new TransformSpec(null, Collections.emptyList())
@@ -2038,6 +2042,8 @@ public class ControllerImpl implements Controller
   /**
    * Compute shard columns for {@link DimensionRangeShardSpec}. Returns an empty list if range-based sharding
    * is not applicable.
+   *
+   * @return pair of shard columns and commentary
    */
   private static Pair<List<String>, String> computeShardColumns(
       final RowSignature signature,
@@ -2048,7 +2054,10 @@ public class ControllerImpl implements Controller
   {
     if (mayHaveMultiValuedClusterByFields) {
       // DimensionRangeShardSpec cannot handle multivalued fields.
-      return Pair.of(Collections.emptyList(), "Cannot use RangeShardSpec, the fields in the CLUSTERED BY clause contains multivalued fields. Using NumberedShardSpec instead.");
+      return Pair.of(
+          Collections.emptyList(),
+          "Cannot use 'range' shard specs since CLUSTERED BY contains multi-valued fields."
+      );
     }
     final List<KeyColumn> clusterByColumns = clusterBy.getColumns();
     final List<String> shardColumns = new ArrayList<>();
@@ -2056,7 +2065,7 @@ public class ControllerImpl implements Controller
     final int numShardColumns = clusterByColumns.size() - clusterBy.getBucketByCount() - (boosted ? 1 : 0);
 
     if (numShardColumns == 0) {
-      return Pair.of(Collections.emptyList(), "Using NumberedShardSpec as no columns are supplied in the 'CLUSTERED BY' clause.");
+      return Pair.of(Collections.emptyList(), "CLUSTERED BY clause is empty.");
     }
 
     for (int i = clusterBy.getBucketByCount(); i < clusterBy.getBucketByCount() + numShardColumns; i++) {
@@ -2065,25 +2074,47 @@ public class ControllerImpl implements Controller
 
       // DimensionRangeShardSpec only handles ascending order.
       if (column.order() != KeyOrder.ASCENDING) {
-        return Pair.of(Collections.emptyList(), "Cannot use RangeShardSpec, RangedShardSpec only supports ascending CLUSTER BY keys. Using NumberedShardSpec instead.");
+        return Pair.of(
+            shardColumns,
+            StringUtils.format(
+                "Using[%d] CLUSTERED BY columns for 'range' shard specs, since the next column has order[%s].",
+                shardColumns.size(),
+                column.order()
+            )
+        );
       }
 
       ColumnType columnType = signature.getColumnType(column.columnName()).orElse(null);
 
       // DimensionRangeShardSpec only handles strings.
       if (!(ColumnType.STRING.equals(columnType))) {
-        return Pair.of(Collections.emptyList(), "Cannot use RangeShardSpec, RangedShardSpec only supports string CLUSTER BY keys. Using NumberedShardSpec instead.");
+        return Pair.of(
+            shardColumns,
+            StringUtils.format(
+                "Using[%d] CLUSTERED BY columns for 'range' shard specs, since the next column is of type[%s]. "
+                + "Only string columns are included in 'range' shard specs.",
+                shardColumns.size(),
+                columnType
+            )
+        );
       }
 
       // DimensionRangeShardSpec only handles columns that appear as-is in the output.
       if (outputColumns.isEmpty()) {
-        return Pair.of(Collections.emptyList(), StringUtils.format("Cannot use RangeShardSpec, Could not find output column name for column [%s]. Using NumberedShardSpec instead.", column.columnName()));
+        return Pair.of(
+            shardColumns,
+            StringUtils.format(
+                "Using only[%d] CLUSTERED BY columns for 'range' shard specs, since the next column was not mapped to "
+                + "an output column.",
+                shardColumns.size()
+            )
+        );
       }
 
       shardColumns.add(columnMappings.getOutputColumnName(outputColumns.getInt(0)));
     }
 
-    return Pair.of(shardColumns, "Using RangeShardSpec to generate segments.");
+    return Pair.of(shardColumns, "Using 'range' shard specs with all CLUSTERED BY fields.");
   }
 
   /**
@@ -2102,22 +2133,15 @@ public class ControllerImpl implements Controller
   private static StringTuple makeStringTuple(
       final ClusterBy clusterBy,
       final RowKeyReader keyReader,
-      final RowKey key
+      final RowKey key,
+      final int shardFieldCount
   )
   {
     final String[] array = new String[clusterBy.getColumns().size() - clusterBy.getBucketByCount()];
-    final boolean boosted = isClusterByBoosted(clusterBy);
 
-    for (int i = 0; i < array.length; i++) {
+    for (int i = 0; i < shardFieldCount; i++) {
       final Object val = keyReader.read(key, clusterBy.getBucketByCount() + i);
-
-      if (i == array.length - 1 && boosted) {
-        // Boost column
-        //noinspection RedundantCast: false alarm; the cast is necessary
-        array[i] = StringUtils.format("%016d", (long) val);
-      } else {
-        array[i] = (String) val;
-      }
+      array[i] = (String) val;
     }
 
     return new StringTuple(array);
@@ -2133,7 +2157,8 @@ public class ControllerImpl implements Controller
     if (dimensionSchemas != null && dimensionSchemas.containsKey(outputColumnName)) {
       return dimensionSchemas.get(outputColumnName);
     }
-    // In case of ingestion, or when metrics are converted to dimensions when compaction is performed without rollup,
+    // In case of ingestion, or when metrics are converted to dimensions when compaction is performed without rollu
+
     // we won't have an entry in the map. For those cases, use the default config.
     return DimensionSchemaUtils.createDimensionSchema(
         outputColumnName,
@@ -2143,10 +2168,11 @@ public class ControllerImpl implements Controller
     );
   }
 
-  private static Pair<List<DimensionSchema>, List<AggregatorFactory>> makeDimensionsAndAggregatorsForIngestion(
+  private static Pair<DimensionsSpec, List<AggregatorFactory>> makeDimensionsAndAggregatorsForIngestion(
       final RowSignature querySignature,
       final ClusterBy queryClusterBy,
-      final List<String> segmentSortOrder,
+      final List<String> contextSegmentSortOrder,
+      final boolean forceSegmentSortByTime,
       final ColumnMappings columnMappings,
       final boolean isRollupQuery,
       final Query<?> query,
@@ -2176,7 +2202,12 @@ public class ControllerImpl implements Controller
     // that order.
 
     // Start with segmentSortOrder.
-    final Set<String> outputColumnsInOrder = new LinkedHashSet<>(segmentSortOrder);
+    final Set<String> outputColumnsInOrder = new LinkedHashSet<>(contextSegmentSortOrder);
+
+    // Then __time, if it's an output column and forceSegmentSortByTime is set.
+    if (columnMappings.hasOutputColumn(ColumnHolder.TIME_COLUMN_NAME) && forceSegmentSortByTime) {
+      outputColumnsInOrder.add(ColumnHolder.TIME_COLUMN_NAME);
+    }
 
     // Then the query-level CLUSTERED BY.
     // Note: this doesn't work when CLUSTERED BY specifies an expression that is not being selected.
@@ -2211,7 +2242,7 @@ public class ControllerImpl implements Controller
       }
     }
 
-    // Each column can be of either time, dimension, aggregator. For this method. we can ignore the time column.
+    // Each column can be either a dimension or an aggregator.
     // For non-complex columns, If the aggregator factory of the column is not available, we treat the column as
     // a dimension. For complex columns, certains hacks are in place.
     for (final String outputColumnName : outputColumnsInOrder) {
@@ -2226,10 +2257,26 @@ public class ControllerImpl implements Controller
           querySignature.getColumnType(queryColumn)
                         .orElseThrow(() -> new ISE("No type for column [%s]", outputColumnName));
 
-      if (!outputColumnName.equals(ColumnHolder.TIME_COLUMN_NAME)) {
-
-        if (!type.is(ValueType.COMPLEX)) {
-          // non complex columns
+      if (!type.is(ValueType.COMPLEX)) {
+        // non complex columns
+        populateDimensionsAndAggregators(
+            dimensions,
+            aggregators,
+            outputColumnAggregatorFactories,
+            outputColumnName,
+            type,
+            query.context(),
+            dimensionSchemas
+        );
+      } else {
+        // complex columns only
+        if (DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.containsKey(type.getComplexTypeName())) {
+          dimensions.add(
+              getDimensionSchema(outputColumnName, type, query.context(), dimensionSchemas)
+          );
+        } else if (!isRollupQuery) {
+          aggregators.add(new PassthroughAggregatorFactory(outputColumnName, type.getComplexTypeName()));
+        } else {
           populateDimensionsAndAggregators(
               dimensions,
               aggregators,
@@ -2239,32 +2286,23 @@ public class ControllerImpl implements Controller
               query.context(),
               dimensionSchemas
           );
-        } else {
-          // complex columns only
-          if (DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.containsKey(type.getComplexTypeName())) {
-            dimensions.add(
-                getDimensionSchema(outputColumnName, type, query.context(), dimensionSchemas)
-            );
-          } else if (!isRollupQuery) {
-            aggregators.add(new PassthroughAggregatorFactory(outputColumnName, type.getComplexTypeName()));
-          } else {
-            populateDimensionsAndAggregators(
-                dimensions,
-                aggregators,
-                outputColumnAggregatorFactories,
-                outputColumnName,
-                type,
-                query.context(),
-                dimensionSchemas
-            );
-          }
         }
       }
     }
 
-    return Pair.of(dimensions, aggregators);
-  }
+    final DimensionsSpec.Builder dimensionsSpecBuilder = DimensionsSpec.builder();
 
+    if (!dimensions.isEmpty() && dimensions.get(0).getName().equals(ColumnHolder.TIME_COLUMN_NAME)) {
+      // Skip __time if it's in the first position, for compatibility with legacy dimensionSpecs.
+      dimensions.remove(0);
+      dimensionsSpecBuilder.setForceSegmentSortByTime(null);
+    } else {
+      // Store explicit forceSegmentSortByTime only if false, for compatibility with legacy dimensionSpecs.
+      dimensionsSpecBuilder.setForceSegmentSortByTime(forceSegmentSortByTime ? null : false);
+    }
+
+    return Pair.of(dimensionsSpecBuilder.setDimensions(dimensions).build(), aggregators);
+  }
 
   /**
    * If the output column is present in the outputColumnAggregatorFactories that means we already have the aggregator information for this column.
@@ -2286,7 +2324,12 @@ public class ControllerImpl implements Controller
       Map<String, DimensionSchema> dimensionSchemas
   )
   {
-    if (outputColumnAggregatorFactories.containsKey(outputColumn)) {
+    if (ColumnHolder.TIME_COLUMN_NAME.equals(outputColumn)) {
+      if (!type.is(ValueType.LONG)) {
+        throw DruidException.defensive("Incorrect type[%s] for column[%s]", type, outputColumn);
+      }
+      dimensions.add(new LongDimensionSchema(outputColumn));
+    } else if (outputColumnAggregatorFactories.containsKey(outputColumn)) {
       aggregators.add(outputColumnAggregatorFactories.get(outputColumn));
     } else {
       dimensions.add(
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByPreShuffleFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByPreShuffleFrameProcessor.java
index ac568322400..a859ea8cd53 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByPreShuffleFrameProcessor.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByPreShuffleFrameProcessor.java
@@ -55,6 +55,7 @@ import org.apache.druid.query.spec.SpecificSegmentSpec;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.SegmentReference;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.timeline.SegmentId;
 
@@ -147,11 +148,13 @@ public class GroupByPreShuffleFrameProcessor extends BaseLeafFrameProcessor
   {
     if (resultYielder == null) {
       final ResourceHolder<Segment> segmentHolder = closer.register(segment.getOrLoad());
+      final SegmentReference mappedSegment = mapSegment(segmentHolder.get());
 
       final Sequence<ResultRow> rowSequence =
           groupingEngine.process(
               query.withQuerySegmentSpec(new SpecificSegmentSpec(segment.getDescriptor())),
-              mapSegment(segmentHolder.get()).asStorageAdapter(),
+              mappedSegment.asStorageAdapter(),
+              mappedSegment.as(TimeBoundaryInspector.class),
               null
           );
 
@@ -179,11 +182,13 @@ public class GroupByPreShuffleFrameProcessor extends BaseLeafFrameProcessor
       if (inputChannel.canRead()) {
         final Frame frame = inputChannel.read();
         final FrameSegment frameSegment = new FrameSegment(frame, inputFrameReader, SegmentId.dummy("x"));
+        final SegmentReference mappedSegment = mapSegment(frameSegment);
 
         final Sequence<ResultRow> rowSequence =
             groupingEngine.process(
                 query.withQuerySegmentSpec(new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY)),
-                mapSegment(frameSegment).asStorageAdapter(),
+                mappedSegment.asStorageAdapter(),
+                mappedSegment.as(TimeBoundaryInspector.class),
                 null
             );
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
index 7af34a1eb55..60fa197c4a5 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
@@ -237,21 +237,20 @@ public class MSQTaskQueryMaker implements QueryMaker
 
       final List<String> segmentSortOrder = MultiStageQueryContext.getSortOrder(sqlQueryContext);
 
-      MSQTaskQueryMakerUtils.validateSegmentSortOrder(
+      MSQTaskQueryMakerUtils.validateContextSortOrderColumnsExist(
           segmentSortOrder,
-          fieldMapping.stream().map(f -> f.getValue()).collect(Collectors.toList())
+          fieldMapping.stream().map(Entry::getValue).collect(Collectors.toSet())
       );
 
-      final DataSourceMSQDestination dataSourceMSQDestination = new DataSourceMSQDestination(
+      final DataSourceMSQDestination dataSourceDestination = new DataSourceMSQDestination(
           targetDataSource.getDestinationName(),
           segmentGranularityObject,
           segmentSortOrder,
           replaceTimeChunks,
           null
       );
-      MultiStageQueryContext.validateAndGetTaskLockType(sqlQueryContext,
-                                                        dataSourceMSQDestination.isReplaceTimeChunks());
-      destination = dataSourceMSQDestination;
+      MultiStageQueryContext.validateAndGetTaskLockType(sqlQueryContext, dataSourceDestination.isReplaceTimeChunks());
+      destination = dataSourceDestination;
     } else {
       final MSQSelectDestination msqSelectDestination = MultiStageQueryContext.getSelectDestination(sqlQueryContext);
       if (msqSelectDestination.equals(MSQSelectDestination.TASKREPORT)) {
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
index e3baa058b41..d20abffb0f9 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
@@ -23,6 +23,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Inject;
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import org.apache.calcite.rel.RelCollation;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.RelRoot;
 import org.apache.calcite.rel.core.Project;
@@ -33,6 +35,7 @@ import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.schema.Table;
 import org.apache.calcite.sql.dialect.CalciteSqlDialect;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.error.DruidException;
 import org.apache.druid.error.InvalidInput;
 import org.apache.druid.error.InvalidSqlInput;
@@ -43,6 +46,7 @@ import org.apache.druid.msq.querykit.QueryKitUtils;
 import org.apache.druid.msq.util.ArrayIngestMode;
 import org.apache.druid.msq.util.DimensionSchemaUtils;
 import org.apache.druid.msq.util.MultiStageQueryContext;
+import org.apache.druid.query.QueryContext;
 import org.apache.druid.rpc.indexing.OverlordClient;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
@@ -61,7 +65,6 @@ import org.apache.druid.sql.destination.IngestDestination;
 import org.apache.druid.sql.destination.TableDestination;
 
 import javax.annotation.Nullable;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -176,8 +179,7 @@ public class MSQTaskSqlEngine implements SqlEngine
   )
   {
     validateInsert(
-        relRoot.rel,
-        relRoot.fields,
+        relRoot,
         destination instanceof TableDestination
         ? plannerContext.getPlannerToolbox()
                         .rootSchema()
@@ -221,35 +223,38 @@ public class MSQTaskSqlEngine implements SqlEngine
    * Engine-specific validation that happens after the query is planned.
    */
   private static void validateInsert(
-      final RelNode rootRel,
-      final List<Entry<Integer, String>> fieldMappings,
+      final RelRoot relRoot,
       @Nullable Table targetTable,
       final PlannerContext plannerContext
   )
   {
-    final int timeColumnIndex = getTimeColumnIndex(fieldMappings);
+    final Map<String, Integer> outputFieldMap = validateNoDuplicateAliases(relRoot.fields);
+    final int timeColumnIndex = getTimeColumnIndex(outputFieldMap);
     final Granularity segmentGranularity = getSegmentGranularity(plannerContext);
-
-    validateNoDuplicateAliases(fieldMappings);
-    validateTimeColumnType(rootRel, timeColumnIndex);
+    validateTimeColumnType(relRoot.rel, timeColumnIndex);
     validateTimeColumnExistsIfNeeded(timeColumnIndex, segmentGranularity);
-    validateLimitAndOffset(rootRel, Granularities.ALL.equals(segmentGranularity));
-    validateTypeChanges(rootRel, fieldMappings, targetTable, plannerContext);
+    validateLimitAndOffset(relRoot.rel, Granularities.ALL.equals(segmentGranularity));
+    validateTypeChanges(relRoot.rel, relRoot.fields, targetTable, plannerContext);
+    validateSortOrderBeginsWithTimeIfRequired(relRoot.fields, relRoot.collation, plannerContext);
   }
 
   /**
    * SQL allows multiple output columns with the same name. However, we don't allow this for INSERT or REPLACE
    * queries, because we use these output names to generate columns in segments. They must be unique.
+   *
+   * @return map of output alias (rhs of the Pair) to position in the {@link RelRoot#rel} (lhs of the Pair)
    */
-  private static void validateNoDuplicateAliases(final List<Entry<Integer, String>> fieldMappings)
+  private static Map<String, Integer> validateNoDuplicateAliases(final List<Entry<Integer, String>> fieldMappings)
   {
-    final Set<String> aliasesSeen = new HashSet<>();
+    final Map<String, Integer> retVal = new Object2IntOpenHashMap<>();
 
     for (final Entry<Integer, String> field : fieldMappings) {
-      if (!aliasesSeen.add(field.getValue())) {
+      if (retVal.put(field.getValue(), field.getKey()) != null) {
         throw InvalidSqlInput.exception("Duplicate field in SELECT: [%s]", field.getValue());
       }
     }
+
+    return retVal;
   }
 
   /**
@@ -347,7 +352,7 @@ public class MSQTaskSqlEngine implements SqlEngine
    * MVDs and arrays are mixed. So, we detect them as invalid.
    *
    * @param rootRel        root rel
-   * @param fieldMappings  field mappings from {@link #validateInsert(RelNode, List, Table, PlannerContext)}
+   * @param fieldMappings  field mappings from {@link #validateInsert(RelRoot, Table, PlannerContext)}
    * @param targetTable    table we are inserting (or replacing) into, if any
    * @param plannerContext planner context
    */
@@ -422,20 +427,80 @@ public class MSQTaskSqlEngine implements SqlEngine
   }
 
   /**
-   * Returns the index of {@link ColumnHolder#TIME_COLUMN_NAME} within a list of field mappings from
-   * {@link #validateInsert(RelNode, List, Table, PlannerContext)}.
+   * Validate that the sort order given by CLUSTERED BY or {@link MultiStageQueryContext#getSortOrder(QueryContext)}
+   * begins with {@link ColumnHolder#TIME_COLUMN_NAME}, unless {@link MultiStageQueryContext#CTX_FORCE_TIME_SORT}
+   * is set.
    *
-   * Returns -1 if the list does not contain a time column.
+   * @param fieldMappings  field mappings from {@link #validateInsert(RelRoot, Table, PlannerContext)}
+   * @param rootCollation  collation of the root rel. Corresponds to the CLUSTERED BY
+   * @param plannerContext planner context
    */
-  private static int getTimeColumnIndex(final List<Entry<Integer, String>> fieldMappings)
+  private static void validateSortOrderBeginsWithTimeIfRequired(
+      final List<Entry<Integer, String>> fieldMappings,
+      final RelCollation rootCollation,
+      final PlannerContext plannerContext
+  )
   {
-    for (final Entry<Integer, String> field : fieldMappings) {
-      if (field.getValue().equals(ColumnHolder.TIME_COLUMN_NAME)) {
-        return field.getKey();
-      }
+    // Segment sort order is determined by the segmentSortOrder parameter if set. Otherwise it's determined by
+    // the rootCollation.
+
+    final QueryContext context = plannerContext.queryContext();
+
+    if (!MultiStageQueryContext.isForceSegmentSortByTime(context)) {
+      // Any sort order is allowed. Skip check.
+      return;
     }
 
-    return -1;
+    final List<String> contextSortOrder = MultiStageQueryContext.getSortOrder(context);
+
+    if (!contextSortOrder.isEmpty()) {
+      final boolean timeIsFirst = ColumnHolder.TIME_COLUMN_NAME.equals(contextSortOrder.get(0));
+
+      if (!timeIsFirst) {
+        throw InvalidSqlInput.exception(
+            "Context parameter[%s] must start with[%s] unless context parameter[%s] is set to[false]. %s",
+            MultiStageQueryContext.CTX_SORT_ORDER,
+            ColumnHolder.TIME_COLUMN_NAME,
+            MultiStageQueryContext.CTX_FORCE_TIME_SORT,
+            DimensionsSpec.WARNING_NON_TIME_SORT_ORDER
+        );
+      }
+    } else if (!rootCollation.getFieldCollations().isEmpty()) {
+      int timePosition = -1;
+      for (int i = 0; i < rootCollation.getFieldCollations().size(); i++) {
+        final String fieldCollationName =
+            fieldMappings.get(rootCollation.getFieldCollations().get(i).getFieldIndex()).getValue();
+        if (ColumnHolder.TIME_COLUMN_NAME.equals(fieldCollationName)) {
+          timePosition = i;
+          break;
+        }
+      }
+
+      if (timePosition > 0) {
+        throw InvalidSqlInput.exception(
+            "Sort order (CLUSTERED BY) cannot include[%s] in position[%d] unless context parameter[%s] "
+            + "is set to[false]. %s",
+            ColumnHolder.TIME_COLUMN_NAME,
+            timePosition,
+            MultiStageQueryContext.CTX_FORCE_TIME_SORT,
+            DimensionsSpec.WARNING_NON_TIME_SORT_ORDER
+        );
+      }
+    }
+  }
+
+  /**
+   * Returns the index of {@link ColumnHolder#TIME_COLUMN_NAME} within a list of field mappings from
+   * {@link #validateInsert(RelRoot, Table, PlannerContext)}.
+   *
+   * @param outputFieldMapping mapping from {@link #validateNoDuplicateAliases(List)}
+   *
+   * @return field position, or -1 if the list does not contain a time column.
+   */
+  private static int getTimeColumnIndex(final Map<String, Integer> outputFieldMapping)
+  {
+    final Integer position = outputFieldMapping.get(ColumnHolder.TIME_COLUMN_NAME);
+    return position != null ? position : -1;
   }
 
   /**
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MSQTaskQueryMakerUtils.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MSQTaskQueryMakerUtils.java
index ae1558babf5..a30c9bb0aec 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MSQTaskQueryMakerUtils.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MSQTaskQueryMakerUtils.java
@@ -21,15 +21,13 @@ package org.apache.druid.msq.util;
 
 import com.google.common.collect.ImmutableSet;
 import org.apache.druid.error.DruidException;
-import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.error.InvalidSqlInput;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.msq.exec.SegmentSource;
 import org.apache.druid.msq.indexing.MSQControllerTask;
 import org.apache.druid.msq.indexing.MSQSpec;
 import org.apache.druid.msq.indexing.destination.DataSourceMSQDestination;
-import org.apache.druid.segment.column.ColumnHolder;
 
-import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -76,24 +74,25 @@ public class MSQTaskQueryMakerUtils
   }
 
   /**
-   * Validates if each element of the sort order appears in the final output and if it is not empty then it starts with the
-   * __time column
+   * Validates that each element of {@link MultiStageQueryContext#CTX_SORT_ORDER}, if provided, appears in the
+   * final output.
    */
-  public static void validateSegmentSortOrder(final List<String> sortOrder, final Collection<String> allOutputColumns)
+  public static void validateContextSortOrderColumnsExist(
+      final List<String> contextSortOrder,
+      final Set<String> allOutputColumns
+  )
   {
     final Set<String> allOutputColumnsSet = new HashSet<>(allOutputColumns);
 
-    for (final String column : sortOrder) {
+    for (final String column : contextSortOrder) {
       if (!allOutputColumnsSet.contains(column)) {
-        throw new IAE("Column [%s] in segment sort order does not appear in the query output", column);
+        throw InvalidSqlInput.exception(
+            "Column[%s] from context parameter[%s] does not appear in the query output",
+            column,
+            MultiStageQueryContext.CTX_SORT_ORDER
+        );
       }
     }
-
-    if (sortOrder.size() > 0
-        && allOutputColumns.contains(ColumnHolder.TIME_COLUMN_NAME)
-        && !ColumnHolder.TIME_COLUMN_NAME.equals(sortOrder.get(0))) {
-      throw new IAE("Segment sort order must begin with column [%s]", ColumnHolder.TIME_COLUMN_NAME);
-    }
   }
 
   /**
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java
index 6810b54457a..9b715f8c8cf 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.opencsv.RFC4180Parser;
 import com.opencsv.RFC4180ParserBuilder;
+import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.error.DruidException;
 import org.apache.druid.indexing.common.TaskLockType;
 import org.apache.druid.indexing.common.task.Tasks;
@@ -167,6 +168,9 @@ public class MultiStageQueryContext
   public static final String CTX_ARRAY_INGEST_MODE = "arrayIngestMode";
   public static final ArrayIngestMode DEFAULT_ARRAY_INGEST_MODE = ArrayIngestMode.ARRAY;
 
+  public static final String CTX_FORCE_TIME_SORT = DimensionsSpec.PARAMETER_FORCE_TIME_SORT;
+  private static final boolean DEFAULT_FORCE_TIME_SORT = DimensionsSpec.DEFAULT_FORCE_TIME_SORT;
+
   public static final String MAX_ROWS_MATERIALIZED_IN_WINDOW = "maxRowsMaterializedInWindow";
 
   public static final String CTX_SKIP_TYPE_VERIFICATION = "skipTypeVerification";
@@ -361,6 +365,11 @@ public class MultiStageQueryContext
     return queryContext.getEnum(CTX_ARRAY_INGEST_MODE, ArrayIngestMode.class, DEFAULT_ARRAY_INGEST_MODE);
   }
 
+  public static boolean isForceSegmentSortByTime(final QueryContext queryContext)
+  {
+    return queryContext.getBoolean(CTX_FORCE_TIME_SORT, DEFAULT_FORCE_TIME_SORT);
+  }
+
   public static Set<String> getColumnsExcludedFromTypeVerification(final QueryContext queryContext)
   {
     return new HashSet<>(decodeList(CTX_SKIP_TYPE_VERIFICATION, queryContext.getString(CTX_SKIP_TYPE_VERIFICATION)));
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
index cbdddf4a46e..96da93c34d0 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
@@ -76,6 +76,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -103,6 +104,7 @@ public class MSQReplaceTest extends MSQTestBase
         };
     return Arrays.asList(data);
   }
+
   @MethodSource("data")
   @ParameterizedTest(name = "{index}:with context {0}")
   public void testReplaceOnFooWithAll(String contextName, Map<String, Object> context)
@@ -129,10 +131,7 @@ public class MSQReplaceTest extends MSQTestBase
     Mockito.doCallRealMethod()
            .doReturn(ImmutableSet.of(existingDataSegment0, existingDataSegment1))
            .when(testTaskActionClient)
-           .submit(new RetrieveUsedSegmentsAction(
-               EasyMock.eq("foo"),
-               EasyMock.eq(ImmutableList.of(Intervals.ETERNITY))
-           ));
+           .submit(new RetrieveUsedSegmentsAction("foo", ImmutableList.of(Intervals.ETERNITY)));
 
     testIngestQuery().setSql(" REPLACE INTO foo OVERWRITE ALL "
                              + "SELECT __time, m1 "
@@ -194,6 +193,378 @@ public class MSQReplaceTest extends MSQTestBase
                      .verifyResults();
   }
 
+  @MethodSource("data")
+  @ParameterizedTest(name = "{index}:with context {0}")
+  public void testReplaceOnFooWithAllClusteredByDim(String contextName, Map<String, Object> context)
+  {
+    // Tests [CLUSTERED BY dim1] with the default forceSegmentSortByTime (true). In this case,
+    // partitioning uses [dim1] but segment sort uses [__time, dim1].
+    RowSignature rowSignature = RowSignature.builder()
+                                            .add("__time", ColumnType.LONG)
+                                            .add("dim1", ColumnType.STRING)
+                                            .add("m1", ColumnType.FLOAT)
+                                            .build();
+
+    DataSegment existingDataSegment0 = DataSegment.builder()
+                                                  .interval(Intervals.of("2000-01-01T/2000-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    DataSegment existingDataSegment1 = DataSegment.builder()
+                                                  .interval(Intervals.of("2001-01-01T/2001-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    Mockito.doCallRealMethod()
+           .doReturn(ImmutableSet.of(existingDataSegment0, existingDataSegment1))
+           .when(testTaskActionClient)
+           .submit(new RetrieveUsedSegmentsAction("foo", ImmutableList.of(Intervals.ETERNITY)));
+
+    testIngestQuery().setSql(" REPLACE INTO foo OVERWRITE ALL "
+                             + "SELECT __time, dim1, m1 "
+                             + "FROM foo "
+                             + "PARTITIONED BY ALL "
+                             + "CLUSTERED BY dim1")
+                     .setExpectedDataSource("foo")
+                     .setExpectedRowSignature(rowSignature)
+                     .setQueryContext(context)
+                     .setExpectedDestinationIntervals(Intervals.ONLY_ETERNITY)
+                     .setExpectedSegments(
+                         ImmutableSet.of(
+                             SegmentId.of("foo", Intervals.ETERNITY, "test", 0)
+                         )
+                     )
+                     .setExpectedShardSpec(DimensionRangeShardSpec.class)
+                     .setExpectedResultRows(
+                         ImmutableList.of(
+                             new Object[]{946684800000L, NullHandling.sqlCompatible() ? "" : null, 1.0f},
+                             new Object[]{946771200000L, "10.1", 2.0f},
+                             new Object[]{946857600000L, "2", 3.0f},
+                             new Object[]{978307200000L, "1", 4.0f},
+                             new Object[]{978393600000L, "def", 5.0f},
+                             new Object[]{978480000000L, "abc", 6.0f}
+                         )
+                     )
+                     .setExpectedSegmentGenerationProgressCountersForStageWorker(
+                         CounterSnapshotMatcher
+                             .with().segmentRowsProcessed(6),
+                         1, 0
+                     )
+                     .setExpectedLastCompactionState(
+                         expectedCompactionState(
+                             context,
+                             Collections.singletonList("dim1"),
+                             DimensionsSpec.builder()
+                                           .setDimensions(
+                                               ImmutableList.of(
+                                                   new StringDimensionSchema("dim1"),
+                                                   new FloatDimensionSchema("m1")
+                                               )
+                                           )
+                                           .setDimensionExclusions(Collections.singletonList("__time"))
+                                           .build(),
+                             GranularityType.ALL,
+                             Intervals.ETERNITY
+                         )
+                     )
+                     .verifyResults();
+  }
+
+  @MethodSource("data")
+  @ParameterizedTest(name = "{index}:with context {0}")
+  public void testReplaceOnFooWithAllClusteredByDimExplicitSort(String contextName, Map<String, Object> context)
+  {
+    RowSignature rowSignature = RowSignature.builder()
+                                            .add("__time", ColumnType.LONG)
+                                            .add("dim1", ColumnType.STRING)
+                                            .add("m1", ColumnType.FLOAT)
+                                            .build();
+
+    DataSegment existingDataSegment0 = DataSegment.builder()
+                                                  .interval(Intervals.of("2000-01-01T/2000-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    DataSegment existingDataSegment1 = DataSegment.builder()
+                                                  .interval(Intervals.of("2001-01-01T/2001-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    Map<String, Object> queryContext = new HashMap<>(context);
+    queryContext.put(DimensionsSpec.PARAMETER_FORCE_TIME_SORT, false);
+
+    Mockito.doCallRealMethod()
+           .doReturn(ImmutableSet.of(existingDataSegment0, existingDataSegment1))
+           .when(testTaskActionClient)
+           .submit(new RetrieveUsedSegmentsAction("foo", ImmutableList.of(Intervals.ETERNITY)));
+
+    testIngestQuery().setSql(" REPLACE INTO foo OVERWRITE ALL "
+                             + "SELECT __time, dim1, m1 "
+                             + "FROM foo "
+                             + "PARTITIONED BY ALL "
+                             + "CLUSTERED BY dim1")
+                     .setExpectedDataSource("foo")
+                     .setExpectedRowSignature(rowSignature)
+                     .setQueryContext(queryContext)
+                     .setExpectedDestinationIntervals(Intervals.ONLY_ETERNITY)
+                     .setExpectedSegments(
+                         ImmutableSet.of(
+                             SegmentId.of("foo", Intervals.ETERNITY, "test", 0)
+                         )
+                     )
+                     .setExpectedShardSpec(DimensionRangeShardSpec.class)
+                     .setExpectedResultRows(
+                         ImmutableList.of(
+                             new Object[]{946684800000L, NullHandling.sqlCompatible() ? "" : null, 1.0f},
+                             new Object[]{978307200000L, "1", 4.0f},
+                             new Object[]{946771200000L, "10.1", 2.0f},
+                             new Object[]{946857600000L, "2", 3.0f},
+                             new Object[]{978480000000L, "abc", 6.0f},
+                             new Object[]{978393600000L, "def", 5.0f}
+                         )
+                     )
+                     .setExpectedSegmentGenerationProgressCountersForStageWorker(
+                         CounterSnapshotMatcher
+                             .with().segmentRowsProcessed(6),
+                         1, 0
+                     )
+                     .setExpectedLastCompactionState(
+                         expectedCompactionState(
+                             context,
+                             Collections.singletonList("dim1"),
+                             DimensionsSpec.builder()
+                                           .setDimensions(
+                                               ImmutableList.of(
+                                                   new StringDimensionSchema("dim1"),
+                                                   new LongDimensionSchema("__time"),
+                                                   new FloatDimensionSchema("m1")
+                                               )
+                                           )
+                                           .setForceSegmentSortByTime(false)
+                                           .build(),
+                             GranularityType.ALL,
+                             Intervals.ETERNITY
+                         )
+                     )
+                     .verifyResults();
+  }
+
+  @MethodSource("data")
+  @ParameterizedTest(name = "{index}:with context {0}")
+  public void testReplaceOnFooWithAllClusteredByDimThenTimeExplicitSort(String contextName, Map<String, Object> context)
+  {
+    // Tests that [CLUSTERED BY dim1, __time] and [CLUSTERED BY dim1] are same when
+    // forceSegmentSortByTime = false. (Same expectations as the prior test,
+    // testReplaceOnFooWithAllClusteredByDimExplicitSort.)
+    RowSignature rowSignature = RowSignature.builder()
+                                            .add("__time", ColumnType.LONG)
+                                            .add("dim1", ColumnType.STRING)
+                                            .add("m1", ColumnType.FLOAT)
+                                            .build();
+
+    DataSegment existingDataSegment0 = DataSegment.builder()
+                                                  .interval(Intervals.of("2000-01-01T/2000-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    DataSegment existingDataSegment1 = DataSegment.builder()
+                                                  .interval(Intervals.of("2001-01-01T/2001-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    Map<String, Object> queryContext = new HashMap<>(context);
+    queryContext.put(DimensionsSpec.PARAMETER_FORCE_TIME_SORT, false);
+
+    Mockito.doCallRealMethod()
+           .doReturn(ImmutableSet.of(existingDataSegment0, existingDataSegment1))
+           .when(testTaskActionClient)
+           .submit(new RetrieveUsedSegmentsAction("foo", ImmutableList.of(Intervals.ETERNITY)));
+
+    testIngestQuery().setSql(" REPLACE INTO foo OVERWRITE ALL "
+                             + "SELECT __time, dim1, m1 "
+                             + "FROM foo "
+                             + "PARTITIONED BY ALL "
+                             + "CLUSTERED BY dim1, __time")
+                     .setExpectedDataSource("foo")
+                     .setExpectedRowSignature(rowSignature)
+                     .setQueryContext(queryContext)
+                     .setExpectedDestinationIntervals(Intervals.ONLY_ETERNITY)
+                     .setExpectedSegments(
+                         ImmutableSet.of(
+                             SegmentId.of("foo", Intervals.ETERNITY, "test", 0)
+                         )
+                     )
+                     .setExpectedShardSpec(DimensionRangeShardSpec.class)
+                     .setExpectedResultRows(
+                         ImmutableList.of(
+                             new Object[]{946684800000L, NullHandling.sqlCompatible() ? "" : null, 1.0f},
+                             new Object[]{978307200000L, "1", 4.0f},
+                             new Object[]{946771200000L, "10.1", 2.0f},
+                             new Object[]{946857600000L, "2", 3.0f},
+                             new Object[]{978480000000L, "abc", 6.0f},
+                             new Object[]{978393600000L, "def", 5.0f}
+                         )
+                     )
+                     .setExpectedSegmentGenerationProgressCountersForStageWorker(
+                         CounterSnapshotMatcher
+                             .with().segmentRowsProcessed(6),
+                         1, 0
+                     )
+                     .setExpectedLastCompactionState(
+                         expectedCompactionState(
+                             context,
+                             Collections.singletonList("dim1"),
+                             DimensionsSpec.builder()
+                                           .setDimensions(
+                                               ImmutableList.of(
+                                                   new StringDimensionSchema("dim1"),
+                                                   new LongDimensionSchema("__time"),
+                                                   new FloatDimensionSchema("m1")
+                                               )
+                                           )
+                                           .setForceSegmentSortByTime(false)
+                                           .build(),
+                             GranularityType.ALL,
+                             Intervals.ETERNITY
+                         )
+                     )
+                     .verifyResults();
+  }
+
+  @MethodSource("data")
+  @ParameterizedTest(name = "{index}:with context {0}")
+  public void testReplaceOnFooWithAllClusteredByDimThenTimeError(String contextName, Map<String, Object> context)
+  {
+    // Tests that [CLUSTERED BY dim1, __time] is an error when forceSegmentSortByTime = true (the default).
+    testIngestQuery().setSql(" REPLACE INTO foo OVERWRITE ALL "
+                             + "SELECT __time, dim1, m1 "
+                             + "FROM foo "
+                             + "PARTITIONED BY ALL "
+                             + "CLUSTERED BY dim1, __time")
+                     .setExpectedDataSource("foo")
+                     .setQueryContext(context)
+                     .setExpectedValidationErrorMatcher(invalidSqlContains(
+                         "Sort order (CLUSTERED BY) cannot include[__time] in position[1] unless context "
+                         + "parameter[forceSegmentSortByTime] is set to[false]."
+                     ))
+                     .verifyPlanningErrors();
+  }
+
+  @MethodSource("data")
+  @ParameterizedTest(name = "{index}:with context {0}")
+  public void testReplaceOnFooWithAllClusteredByDimThenTimeError2(String contextName, Map<String, Object> context)
+  {
+    // Tests that setting segmentSortOrder = [dim1, __time] is an error when
+    // forceSegmentSortByTime = false (the default).
+    Map<String, Object> queryContext = new HashMap<>(context);
+    queryContext.put(MultiStageQueryContext.CTX_SORT_ORDER, "dim1, __time");
+
+    testIngestQuery().setSql(" REPLACE INTO foo OVERWRITE ALL "
+                             + "SELECT __time, dim1, m1 "
+                             + "FROM foo "
+                             + "PARTITIONED BY ALL "
+                             + "CLUSTERED BY dim1")
+                     .setExpectedDataSource("foo")
+                     .setQueryContext(queryContext)
+                     .setExpectedValidationErrorMatcher(invalidSqlContains(
+                         "Context parameter[segmentSortOrder] must start with[__time] unless context "
+                         + "parameter[forceSegmentSortByTime] is set to[false]."
+                     ))
+                     .verifyPlanningErrors();
+  }
+
+  @MethodSource("data")
+  @ParameterizedTest(name = "{index}:with context {0}")
+  public void testReplaceOnFooWithAllClusteredByTimeThenDimExplicitSort(String contextName, Map<String, Object> context)
+  {
+    // Tests [CLUSTERED BY __time, dim1] with forceSegmentSortByTime = false.
+    RowSignature rowSignature = RowSignature.builder()
+                                            .add("__time", ColumnType.LONG)
+                                            .add("dim1", ColumnType.STRING)
+                                            .add("m1", ColumnType.FLOAT)
+                                            .build();
+
+    DataSegment existingDataSegment0 = DataSegment.builder()
+                                                  .interval(Intervals.of("2000-01-01T/2000-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    DataSegment existingDataSegment1 = DataSegment.builder()
+                                                  .interval(Intervals.of("2001-01-01T/2001-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    Map<String, Object> queryContext = new HashMap<>(context);
+    queryContext.put(DimensionsSpec.PARAMETER_FORCE_TIME_SORT, false);
+
+    Mockito.doCallRealMethod()
+           .doReturn(ImmutableSet.of(existingDataSegment0, existingDataSegment1))
+           .when(testTaskActionClient)
+           .submit(new RetrieveUsedSegmentsAction("foo", ImmutableList.of(Intervals.ETERNITY)));
+
+    testIngestQuery().setSql(" REPLACE INTO foo OVERWRITE ALL "
+                             + "SELECT __time, dim1, m1 "
+                             + "FROM foo "
+                             + "PARTITIONED BY ALL "
+                             + "CLUSTERED BY __time, dim1")
+                     .setExpectedDataSource("foo")
+                     .setExpectedRowSignature(rowSignature)
+                     .setQueryContext(queryContext)
+                     .setExpectedDestinationIntervals(Intervals.ONLY_ETERNITY)
+                     .setExpectedSegments(
+                         ImmutableSet.of(
+                             SegmentId.of("foo", Intervals.ETERNITY, "test", 0)
+                         )
+                     )
+                     .setExpectedShardSpec(NumberedShardSpec.class)
+                     .setExpectedResultRows(
+                         ImmutableList.of(
+                             new Object[]{946684800000L, NullHandling.sqlCompatible() ? "" : null, 1.0f},
+                             new Object[]{946771200000L, "10.1", 2.0f},
+                             new Object[]{946857600000L, "2", 3.0f},
+                             new Object[]{978307200000L, "1", 4.0f},
+                             new Object[]{978393600000L, "def", 5.0f},
+                             new Object[]{978480000000L, "abc", 6.0f}
+                         )
+                     )
+                     .setExpectedSegmentGenerationProgressCountersForStageWorker(
+                         CounterSnapshotMatcher
+                             .with().segmentRowsProcessed(6),
+                         1, 0
+                     )
+                     .setExpectedLastCompactionState(
+                         expectedCompactionState(
+                             context,
+                             Collections.emptyList(),
+                             // For backwards-compatibility, compaction state is stored as if
+                             // forceSegmentSortByTime = true.
+                             ImmutableList.of(
+                                 new StringDimensionSchema("dim1"),
+                                 new FloatDimensionSchema("m1")
+                             ),
+                             GranularityType.ALL,
+                             Intervals.ETERNITY
+                         )
+                     )
+                     .verifyResults();
+  }
+
   @MethodSource("data")
   @ParameterizedTest(name = "{index}:with context {0}")
   public void testReplaceOnFooWithWhere(String contextName, Map<String, Object> context)
@@ -281,7 +652,8 @@ public class MSQReplaceTest extends MSQTestBase
                      .setExpectedMSQSegmentReport(
                          new MSQSegmentReport(
                              NumberedShardSpec.class.getSimpleName(),
-                             "Cannot use RangeShardSpec, RangedShardSpec only supports string CLUSTER BY keys. Using NumberedShardSpec instead."
+                             "Using[0] CLUSTERED BY columns for 'range' shard specs, since the next column is of "
+                             + "type[LONG]. Only string columns are included in 'range' shard specs."
                          )
                      )
                      .setExpectedSegments(ImmutableSet.of(
@@ -527,10 +899,7 @@ public class MSQReplaceTest extends MSQTestBase
     Mockito.doCallRealMethod()
            .doReturn(ImmutableSet.of(existingDataSegment0, existingDataSegment1))
            .when(testTaskActionClient)
-           .submit(new RetrieveUsedSegmentsAction(
-               EasyMock.eq("foo"),
-               EasyMock.eq(ImmutableList.of(Intervals.ETERNITY))
-           ));
+           .submit(new RetrieveUsedSegmentsAction("foo", ImmutableList.of(Intervals.ETERNITY)));
 
 
     testIngestQuery().setSql(" REPLACE INTO foo "
@@ -613,10 +982,7 @@ public class MSQReplaceTest extends MSQTestBase
 
     Mockito.doReturn(ImmutableSet.of(existingDataSegment0))
            .when(testTaskActionClient)
-           .submit(new RetrieveUsedSegmentsAction(
-               EasyMock.eq("foo"),
-               EasyMock.eq(ImmutableList.of(Intervals.of("2000-01-01/2000-03-01")))
-           ));
+           .submit(new RetrieveUsedSegmentsAction("foo", ImmutableList.of(Intervals.of("2000-01-01/2000-03-01"))));
 
     testIngestQuery().setSql(" REPLACE INTO foo "
                              + "OVERWRITE WHERE __time >= TIMESTAMP '2000-01-01' AND __time < TIMESTAMP '2000-03-01' "
@@ -960,7 +1326,7 @@ public class MSQReplaceTest extends MSQTestBase
                      .setExpectedMSQSegmentReport(
                          new MSQSegmentReport(
                              DimensionRangeShardSpec.class.getSimpleName(),
-                             "Using RangeShardSpec to generate segments."
+                             "Using 'range' shard specs with all CLUSTERED BY fields."
                          )
                      )
                      .setExpectedLastCompactionState(expectedCompactionState(
@@ -1119,7 +1485,7 @@ public class MSQReplaceTest extends MSQTestBase
                      .setExpectedMSQSegmentReport(
                          new MSQSegmentReport(
                              DimensionRangeShardSpec.class.getSimpleName(),
-                             "Using RangeShardSpec to generate segments."
+                             "Using 'range' shard specs with all CLUSTERED BY fields."
                          )
                      )
                      .setExpectedRowSignature(rowSignature)
@@ -1338,7 +1704,7 @@ public class MSQReplaceTest extends MSQTestBase
                      .setExpectedMSQSegmentReport(
                          new MSQSegmentReport(
                              NumberedShardSpec.class.getSimpleName(),
-                             "Using NumberedShardSpec as no columns are supplied in the 'CLUSTERED BY' clause."
+                             "CLUSTERED BY clause is empty."
                          )
                      )
                      .setExpectedResultRows(
@@ -2168,6 +2534,25 @@ public class MSQReplaceTest extends MSQTestBase
       GranularityType segmentGranularity,
       Interval interval
   )
+  {
+    return expectedCompactionState(
+        context,
+        partitionDimensions,
+        new DimensionsSpec.Builder().setDimensions(dimensions)
+                                    .setDimensionExclusions(Collections.singletonList("__time"))
+                                    .build(),
+        segmentGranularity,
+        interval
+    );
+  }
+
+  private CompactionState expectedCompactionState(
+      Map<String, Object> context,
+      List<String> partitionDimensions,
+      DimensionsSpec dimensionsSpec,
+      GranularityType segmentGranularity,
+      Interval interval
+  )
   {
     if (!context.containsKey(Tasks.STORE_COMPACTION_STATE_KEY)
         || !((Boolean) context.get(Tasks.STORE_COMPACTION_STATE_KEY))) {
@@ -2187,11 +2572,6 @@ public class MSQReplaceTest extends MSQTestBase
           false
       );
     }
-    DimensionsSpec dimensionsSpec = new DimensionsSpec.Builder()
-        .setDimensions(dimensions)
-        .setDimensionExclusions(Collections.singletonList(
-            "__time"))
-        .build();
 
     IndexSpec indexSpec = new IndexSpec(null, null, null, null, null, null, null);
     GranularitySpec granularitySpec = new UniformGranularitySpec(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/TaskDataSegmentProviderTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/TaskDataSegmentProviderTest.java
index 60d40c6d4f1..b5141e12dc8 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/TaskDataSegmentProviderTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/TaskDataSegmentProviderTest.java
@@ -42,6 +42,7 @@ import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.jackson.JacksonUtils;
 import org.apache.druid.java.util.emitter.EmittingLogger;
 import org.apache.druid.msq.counters.ChannelCounters;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.DimensionHandler;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.Metadata;
@@ -334,6 +335,12 @@ public class TaskDataSegmentProviderTest
       return null;
     }
 
+    @Override
+    public List<OrderBy> getOrdering()
+    {
+      return Collections.emptyList();
+    }
+
     @Override
     public void close()
     {
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MSQTaskQueryMakerUtilsTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MSQTaskQueryMakerUtilsTest.java
index 00b76e7ef26..bb0bdcd34d1 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MSQTaskQueryMakerUtilsTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MSQTaskQueryMakerUtilsTest.java
@@ -20,6 +20,8 @@
 package org.apache.druid.msq.util;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.apache.druid.error.DruidException;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -28,25 +30,21 @@ import java.util.Collections;
 public class MSQTaskQueryMakerUtilsTest
 {
   @Test
-  public void testValidateSegmentSortOrder()
+  public void testValidateContextSortOrderColumnsExist()
   {
     // These are all OK, so validateSegmentSortOrder does nothing.
-    MSQTaskQueryMakerUtils.validateSegmentSortOrder(Collections.emptyList(), ImmutableList.of("__time", "a", "b"));
-    MSQTaskQueryMakerUtils.validateSegmentSortOrder(ImmutableList.of("__time"), ImmutableList.of("__time", "a", "b"));
-    MSQTaskQueryMakerUtils.validateSegmentSortOrder(ImmutableList.of("__time", "b"), ImmutableList.of("__time", "a", "b"));
-    MSQTaskQueryMakerUtils.validateSegmentSortOrder(ImmutableList.of("b"), ImmutableList.of("a", "b"));
+    MSQTaskQueryMakerUtils.validateContextSortOrderColumnsExist(Collections.emptyList(), ImmutableSet.of("__time", "a", "b"));
+    MSQTaskQueryMakerUtils.validateContextSortOrderColumnsExist(ImmutableList.of("__time"), ImmutableSet.of("__time", "a", "b"));
+    MSQTaskQueryMakerUtils.validateContextSortOrderColumnsExist(ImmutableList.of("__time", "b"), ImmutableSet.of("__time", "a", "b"));
+    MSQTaskQueryMakerUtils.validateContextSortOrderColumnsExist(ImmutableList.of("b"), ImmutableSet.of("a", "b"));
+    MSQTaskQueryMakerUtils.validateContextSortOrderColumnsExist(ImmutableList.of("b", "__time"), ImmutableSet.of("__time", "a", "b"));
 
     // These are not OK.
     Assert.assertThrows(
-        IllegalArgumentException.class,
-        () -> MSQTaskQueryMakerUtils.validateSegmentSortOrder(ImmutableList.of("c"), ImmutableList.of("a", "b"))
-    );
-
-    Assert.assertThrows(
-        IllegalArgumentException.class,
-        () -> MSQTaskQueryMakerUtils.validateSegmentSortOrder(
-            ImmutableList.of("b", "__time"),
-            ImmutableList.of("__time", "a", "b")
+        DruidException.class,
+        () -> MSQTaskQueryMakerUtils.validateContextSortOrderColumnsExist(
+            ImmutableList.of("c"),
+            ImmutableSet.of("a", "b")
         )
     );
   }
diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceGroupByQueryTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceGroupByQueryTest.java
index a91e6353c75..361c84542ad 100644
--- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceGroupByQueryTest.java
+++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceGroupByQueryTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.query.aggregation.variance;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.apache.druid.data.input.Row;
+import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.granularity.PeriodGranularity;
 import org.apache.druid.query.QueryRunner;
@@ -38,21 +39,24 @@ import org.apache.druid.query.groupby.GroupByQueryRunnerFactory;
 import org.apache.druid.query.groupby.GroupByQueryRunnerTest;
 import org.apache.druid.query.groupby.GroupByQueryRunnerTestHelper;
 import org.apache.druid.query.groupby.ResultRow;
+import org.apache.druid.query.groupby.TestGroupByBuffers;
 import org.apache.druid.query.groupby.having.GreaterThanHavingSpec;
 import org.apache.druid.query.groupby.having.OrHavingSpec;
 import org.apache.druid.query.groupby.orderby.DefaultLimitSpec;
 import org.apache.druid.query.groupby.orderby.OrderByColumnSpec;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.joda.time.Period;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  *
@@ -60,6 +64,9 @@ import java.util.stream.Collectors;
 @RunWith(Parameterized.class)
 public class VarianceGroupByQueryTest extends InitializedNullHandlingTest
 {
+  @MonotonicNonNull
+  private static TestGroupByBuffers BUFFER_POOLS = null;
+
   private final GroupByQueryConfig config;
   private final QueryRunner<Row> runner;
   private final GroupByQueryRunnerFactory factory;
@@ -69,17 +76,26 @@ public class VarianceGroupByQueryTest extends InitializedNullHandlingTest
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> constructorFeeder()
   {
-    return GroupByQueryRunnerTest.constructorFeeder().stream()
-                                 .map(
-                                     constructor ->
-                                         new Object[]{
-                                             constructor[0],
-                                             constructor[1],
-                                             constructor[2],
-                                             constructor[3]
-                                         }
-                                 )
-                                 .collect(Collectors.toList());
+    setUpClass();
+
+    final List<Object[]> constructors = new ArrayList<>();
+    for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) {
+      final GroupByQueryRunnerFactory factory = GroupByQueryRunnerTest.makeQueryRunnerFactory(config, BUFFER_POOLS);
+      for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(factory, false)) {
+        final String testName = StringUtils.format("config=%s, runner=%s", config, runner);
+        constructors.add(new Object[]{testName, config, factory, runner});
+      }
+    }
+
+    return constructors;
+  }
+
+  @BeforeClass
+  public static void setUpClass()
+  {
+    if (BUFFER_POOLS == null) {
+      BUFFER_POOLS = TestGroupByBuffers.createDefault();
+    }
   }
 
   public VarianceGroupByQueryTest(
diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceTopNQueryTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceTopNQueryTest.java
index 7e17e6b5848..b78988701d1 100644
--- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceTopNQueryTest.java
+++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceTopNQueryTest.java
@@ -53,7 +53,7 @@ public class VarianceTopNQueryTest extends InitializedNullHandlingTest
   @Parameterized.Parameters(name = "{0}")
   public static Iterable<Object[]> constructorFeeder()
   {
-    return QueryRunnerTestHelper.transformToConstructionFeeder(TopNQueryRunnerTest.queryRunners());
+    return QueryRunnerTestHelper.transformToConstructionFeeder(TopNQueryRunnerTest.queryRunners(true));
   }
 
   private final QueryRunner<Result<TopNResultValue>> runner;
diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java
index c60874e7111..8a124435faa 100644
--- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java
+++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
+import javax.annotation.Nullable;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -291,7 +292,7 @@ public class IndexGeneratorJob implements Jobby
       Bucket theBucket,
       AggregatorFactory[] aggs,
       HadoopDruidIndexerConfig config,
-      Iterable<String> oldDimOrder,
+      @Nullable Iterable<String> oldDimOrder,
       Map<String, ColumnFormat> oldCapabilities
   )
   {
@@ -313,7 +314,7 @@ public class IndexGeneratorJob implements Jobby
                                             .setUseMaxMemoryEstimates(tuningConfig.isUseMaxMemoryEstimates())
                                             .build();
 
-    if (oldDimOrder != null && !indexSchema.getDimensionsSpec().hasCustomDimensions()) {
+    if (oldDimOrder != null && !indexSchema.getDimensionsSpec().hasFixedDimensions()) {
       newIndex.loadDimensionIterable(oldDimOrder, oldCapabilities);
     }
 
@@ -472,7 +473,7 @@ public class IndexGeneratorJob implements Jobby
     private void flushIndexToContextAndClose(BytesWritable key, IncrementalIndex index, Context context)
         throws IOException, InterruptedException
     {
-      final List<String> dimensions = index.getDimensionNames();
+      final List<String> dimensions = index.getDimensionNames(false);
       Iterator<Row> rows = index.iterator();
       while (rows.hasNext()) {
         context.progress();
@@ -709,7 +710,7 @@ public class IndexGeneratorJob implements Jobby
           ++lineCount;
 
           if (!index.canAppendRow()) {
-            allDimensionNames.addAll(index.getDimensionOrder());
+            allDimensionNames.addAll(index.getDimensionNames(false));
 
             log.info(index.getOutOfRowsReason());
             log.info(
@@ -752,7 +753,7 @@ public class IndexGeneratorJob implements Jobby
                 bucket,
                 combiningAggs,
                 config,
-                allDimensionNames,
+                index.getDimensionOrder(),
                 persistIndex.getColumnFormats()
             );
             startTime = System.currentTimeMillis();
@@ -760,7 +761,7 @@ public class IndexGeneratorJob implements Jobby
           }
         }
 
-        allDimensionNames.addAll(index.getDimensionOrder());
+        allDimensionNames.addAll(index.getDimensionNames(false));
 
         log.info("%,d lines completed.", lineCount);
 
diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/path/DatasourcePathSpec.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/path/DatasourcePathSpec.java
index c56dfaeb089..3f107b937db 100644
--- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/path/DatasourcePathSpec.java
+++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/path/DatasourcePathSpec.java
@@ -128,7 +128,7 @@ public class DatasourcePathSpec implements PathSpec
     DatasourceIngestionSpec updatedIngestionSpec = ingestionSpec;
     if (updatedIngestionSpec.getDimensions() == null) {
       List<String> dims;
-      if (config.getParser().getParseSpec().getDimensionsSpec().hasCustomDimensions()) {
+      if (config.getParser().getParseSpec().getDimensionsSpec().hasFixedDimensions()) {
         dims = config.getParser().getParseSpec().getDimensionsSpec().getDimensionNames();
       } else {
         Set<String> dimSet = Sets.newHashSet(
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java
index 5a17c4379f1..3d5c92c543e 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java
@@ -620,10 +620,19 @@ public abstract class AbstractBatchIndexTask extends AbstractTask
     if (storeCompactionState) {
       TuningConfig tuningConfig = ingestionSpec.getTuningConfig();
       GranularitySpec granularitySpec = ingestionSpec.getDataSchema().getGranularitySpec();
-      // We do not need to store dimensionExclusions and spatialDimensions since auto compaction does not support them
-      DimensionsSpec dimensionsSpec = ingestionSpec.getDataSchema().getDimensionsSpec() == null
-                                      ? null
-                                      : new DimensionsSpec(ingestionSpec.getDataSchema().getDimensionsSpec().getDimensions());
+      DimensionsSpec dimensionsSpec;
+      if (ingestionSpec.getDataSchema().getDimensionsSpec() == null) {
+        dimensionsSpec = null;
+      } else {
+        // We do not need to store spatial dimensions, since by this point they've been converted to regular dimensions.
+        // We also do not need to store dimensionExclusions, only dimensions that exist.
+        final DimensionsSpec inputDimensionsSpec = ingestionSpec.getDataSchema().getDimensionsSpec();
+        dimensionsSpec = DimensionsSpec
+            .builder()
+            .setDimensions(inputDimensionsSpec.getDimensions())
+            .setForceSegmentSortByTime(inputDimensionsSpec.isForceSegmentSortByTimeConfigured())
+            .build();
+      }
       // We only need to store filter since that is the only field auto compaction support
       Map<String, Object> transformSpec = ingestionSpec.getDataSchema().getTransformSpec() == null || TransformSpec.NONE.equals(ingestionSpec.getDataSchema().getTransformSpec())
                                           ? null
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
index 8659eb0f397..ab7736b047e 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
@@ -42,6 +42,7 @@ import org.apache.druid.data.input.SplitHintSpec;
 import org.apache.druid.data.input.impl.DimensionSchema;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.error.InvalidInput;
 import org.apache.druid.indexer.Checks;
 import org.apache.druid.indexer.Property;
@@ -70,8 +71,9 @@ import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.java.util.emitter.service.ServiceEmitter;
 import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.segment.DimensionHandler;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.QueryableIndex;
@@ -109,6 +111,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.function.Supplier;
@@ -753,8 +756,8 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
     private Granularity queryGranularity;
 
     // For processDimensionsSpec:
-    private final BiMap<String, Integer> uniqueDims = HashBiMap.create();
-    private final Map<String, DimensionSchema> dimensionSchemaMap = new HashMap<>();
+    private final BiMap<String, Integer> uniqueDims = HashBiMap.create(); // dimension name -> position in sort order
+    private final Map<String, DimensionSchema> dimensionSchemaMap = new HashMap<>(); // dimension name -> schema
 
     // For processMetricsSpec:
     private final Set<List<AggregatorFactory>> aggregatorFactoryLists = new HashSet<>();
@@ -831,19 +834,34 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
       }
 
       final BiMap<Integer, String> orderedDims = uniqueDims.inverse();
+
+      // Include __time as a dimension only if required, i.e., if it appears in the sort order after position 0.
+      final Integer timePosition = uniqueDims.get(ColumnHolder.TIME_COLUMN_NAME);
+      final boolean includeTimeAsDimension = timePosition != null && timePosition > 0;
+
       final List<DimensionSchema> dimensionSchemas =
           IntStream.range(0, orderedDims.size())
                    .mapToObj(i -> {
                      final String dimName = orderedDims.get(i);
-                     return Preconditions.checkNotNull(
-                         dimensionSchemaMap.get(dimName),
-                         "Cannot find dimension[%s] from dimensionSchemaMap",
-                         dimName
-                     );
+                     if (ColumnHolder.TIME_COLUMN_NAME.equals(dimName) && !includeTimeAsDimension) {
+                       return null;
+                     } else {
+                       return Preconditions.checkNotNull(
+                           dimensionSchemaMap.get(dimName),
+                           "Cannot find dimension[%s] from dimensionSchemaMap",
+                           dimName
+                       );
+                     }
                    })
+                   .filter(Objects::nonNull)
                    .collect(Collectors.toList());
 
-      return new DimensionsSpec(dimensionSchemas);
+      // Store forceSegmentSortByTime only if false, for compatibility with legacy compaction states.
+      final Boolean forceSegmentSortByTime = includeTimeAsDimension ? false : null;
+      return DimensionsSpec.builder()
+          .setDimensions(dimensionSchemas)
+          .setForceSegmentSortByTime(forceSegmentSortByTime)
+          .build();
     }
 
     public AggregatorFactory[] getMetricsSpec()
@@ -923,27 +941,30 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
         return;
       }
 
-      final Map<String, DimensionHandler> dimensionHandlerMap = index.getDimensionHandlers();
+      final List<String> sortOrder = new ArrayList<>();
 
-      for (String dimension : index.getAvailableDimensions()) {
-        final ColumnHolder columnHolder = Preconditions.checkNotNull(
-            index.getColumnHolder(dimension),
-            "Cannot find column for dimension[%s]",
-            dimension
-        );
+      for (final OrderBy orderBy : index.getOrdering()) {
+        final String dimension = orderBy.getColumnName();
+        if (orderBy.getOrder() != Order.ASCENDING) {
+          throw DruidException.defensive("Order[%s] for dimension[%s] not supported", orderBy.getOrder(), dimension);
+        }
+        sortOrder.add(dimension);
+      }
 
-        if (!uniqueDims.containsKey(dimension)) {
-          Preconditions.checkNotNull(
-              dimensionHandlerMap.get(dimension),
-              "Cannot find dimensionHandler for dimension[%s]",
-              dimension
-          );
+      for (String dimension : Iterables.concat(sortOrder, index.getAvailableDimensions())) {
+        uniqueDims.computeIfAbsent(dimension, ignored -> uniqueDims.size());
 
-          uniqueDims.put(dimension, uniqueDims.size());
-          dimensionSchemaMap.put(
-              dimension,
-              columnHolder.getColumnFormat().getColumnSchema(dimension)
-          );
+        if (!dimensionSchemaMap.containsKey(dimension)) {
+          // Possible for sortOrder to contain a dimension that doesn't exist (i.e. if it's 100% nulls).
+          // In this case, omit it from dimensionSchemaMap for now. We'll skip it later if *no* segment has an existing
+          // column for it.
+          final ColumnHolder columnHolder = index.getColumnHolder(dimension);
+          if (columnHolder != null) {
+            dimensionSchemaMap.put(
+                dimension,
+                columnHolder.getColumnFormat().getColumnSchema(dimension)
+            );
+          }
         }
       }
     }
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/input/InputRowSchemas.java b/indexing-service/src/main/java/org/apache/druid/indexing/input/InputRowSchemas.java
index 3a04b71e2f1..a5f95c2e15f 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/input/InputRowSchemas.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/input/InputRowSchemas.java
@@ -79,7 +79,7 @@ public class InputRowSchemas
     final Set<String> transformOutputNames =
         transformSpec.getTransforms().stream().map(Transform::getName).collect(Collectors.toSet());
 
-    if (dimensionsSpec.hasCustomDimensions()) {
+    if (dimensionsSpec.hasFixedDimensions()) {
       // We need an inclusion-based filter.
       final Set<String> inclusions = new HashSet<>();
 
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java
index 5018795cbb4..4e5e8634f18 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java
@@ -196,7 +196,9 @@ public class InputSourceSampler
           Map<String, Object> parsed = new LinkedHashMap<>();
 
           parsed.put(ColumnHolder.TIME_COLUMN_NAME, row.getTimestampFromEpoch());
-          columnNames.forEach(k -> parsed.put(k, row.getRaw(k)));
+          columnNames.stream()
+                     .filter(k -> !ColumnHolder.TIME_COLUMN_NAME.equals(k))
+                     .forEach(k -> parsed.put(k, row.getRaw(k)));
 
           Number sortKey = row.getMetric(SamplerInputRow.SAMPLER_ORDERING_COLUMN);
           if (sortKey != null) {
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java
index 6fbf9e5deed..307a44dbf7e 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.jsontype.NamedType;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Ordering;
 import com.google.common.io.Files;
 import com.google.common.util.concurrent.Futures;
@@ -34,6 +35,7 @@ import org.apache.druid.client.indexing.ClientCompactionTaskTransformSpec;
 import org.apache.druid.client.indexing.NoopOverlordClient;
 import org.apache.druid.data.input.impl.CSVParseSpec;
 import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.LongDimensionSchema;
 import org.apache.druid.data.input.impl.NewSpatialDimensionSchema;
 import org.apache.druid.data.input.impl.ParseSpec;
 import org.apache.druid.data.input.impl.StringDimensionSchema;
@@ -61,6 +63,7 @@ import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Comparators;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@@ -76,6 +79,7 @@ import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DataSegmentsWithSchemas;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.TestIndex;
 import org.apache.druid.segment.column.ColumnType;
@@ -1865,6 +1869,161 @@ public class CompactionTaskRunTest extends IngestionTestBase
     Assert.assertEquals(rows, rowsFromSegment);
   }
 
+  @Test
+  public void testRunWithAutoCastDimensionsSortByDimension() throws Exception
+  {
+    // Compaction will produce one segment sorted by [x, __time], even though input rows are sorted by __time.
+    final List<String> rows = ImmutableList.of(
+        "2014-01-01T00:00:10Z,a,10,100,1\n",
+        "2014-01-01T00:00:10Z,b,20,110,2\n",
+        "2014-01-01T00:00:10Z,c,30,120,3\n",
+        "2014-01-01T00:01:20Z,a,10,100,1\n",
+        "2014-01-01T00:01:20Z,b,20,110,2\n",
+        "2014-01-01T00:01:20Z,c,30,120,3\n"
+    );
+    final ParseSpec spec = new CSVParseSpec(
+        new TimestampSpec("ts", "auto", null),
+        DimensionsSpec.builder()
+                      .setDimensions(Arrays.asList(
+                          new AutoTypeColumnSchema("x", ColumnType.LONG),
+                          new LongDimensionSchema("__time"),
+                          new AutoTypeColumnSchema("ts", ColumnType.STRING),
+                          new AutoTypeColumnSchema("dim", null),
+                          new AutoTypeColumnSchema("y", ColumnType.LONG)
+                      ))
+                      .setForceSegmentSortByTime(false)
+                      .build(),
+        "|",
+        Arrays.asList("ts", "dim", "x", "y", "val"),
+        false,
+        0
+    );
+    Pair<TaskStatus, DataSegmentsWithSchemas> indexTaskResult = runIndexTask(null, null, spec, rows, false);
+    verifySchema(indexTaskResult.rhs);
+
+    final Builder builder = new Builder(
+        DATA_SOURCE,
+        segmentCacheManagerFactory
+    );
+
+    final CompactionTask compactionTask = builder
+        .interval(Intervals.of("2014-01-01/2014-01-02"))
+        .build();
+
+    final Pair<TaskStatus, DataSegmentsWithSchemas> resultPair = runTask(compactionTask);
+    verifySchema(resultPair.rhs);
+
+    Assert.assertTrue(resultPair.lhs.isSuccess());
+
+    final DataSegmentsWithSchemas dataSegmentsWithSchemas = resultPair.rhs;
+    final List<DataSegment> segments = new ArrayList<>(dataSegmentsWithSchemas.getSegments());
+    Assert.assertEquals(1, segments.size());
+
+    final DataSegment compactSegment = Iterables.getOnlyElement(segments);
+    Assert.assertEquals(
+        Intervals.of("2014-01-01T00:00:00/2014-01-01T01:00:00"),
+        compactSegment.getInterval()
+    );
+    Map<String, String> expectedLongSumMetric = new HashMap<>();
+    expectedLongSumMetric.put("name", "val");
+    expectedLongSumMetric.put("type", "longSum");
+    expectedLongSumMetric.put("fieldName", "val");
+    Assert.assertEquals(
+        getDefaultCompactionState(
+            Granularities.HOUR,
+            Granularities.MINUTE,
+            ImmutableList.of(Intervals.of("2014-01-01T00:00:00/2014-01-01T01:00:00")),
+            DimensionsSpec.builder()
+                          .setDimensions(Arrays.asList(
+                              // check explicitly that time ordering is preserved
+                              new AutoTypeColumnSchema("x", ColumnType.LONG),
+                              new LongDimensionSchema("__time"),
+                              new AutoTypeColumnSchema("ts", ColumnType.STRING),
+                              new AutoTypeColumnSchema("dim", null),
+                              new AutoTypeColumnSchema("y", ColumnType.LONG)
+                          ))
+                          .setForceSegmentSortByTime(false)
+                          .build(),
+            expectedLongSumMetric
+        ),
+        compactSegment.getLastCompactionState()
+    );
+    if (lockGranularity == LockGranularity.SEGMENT) {
+      Assert.assertEquals(
+          new NumberedOverwriteShardSpec(32768, 0, 3, (short) 1, (short) 1),
+          compactSegment.getShardSpec()
+      );
+    } else {
+      Assert.assertEquals(new NumberedShardSpec(0, 1), compactSegment.getShardSpec());
+    }
+
+    final File cacheDir = temporaryFolder.newFolder();
+    final SegmentCacheManager segmentCacheManager = segmentCacheManagerFactory.manufacturate(cacheDir);
+
+    List<String> rowsFromSegment = new ArrayList<>();
+    final File segmentFile = segmentCacheManager.getSegmentFiles(compactSegment);
+
+    final QueryableIndex queryableIndex = testUtils.getTestIndexIO().loadIndex(segmentFile);
+    final WindowedStorageAdapter adapter = new WindowedStorageAdapter(
+        new QueryableIndexStorageAdapter(queryableIndex),
+        compactSegment.getInterval()
+    );
+    Assert.assertEquals(
+        ImmutableList.of(
+            OrderBy.ascending("x"),
+            OrderBy.ascending("__time"),
+            OrderBy.ascending("ts"),
+            OrderBy.ascending("dim"),
+            OrderBy.ascending("y")
+        ),
+        queryableIndex.getOrdering()
+    );
+
+    try (final CursorHolder cursorHolder =
+             adapter.getAdapter()
+                    .makeCursorHolder(CursorBuildSpec.builder().setInterval(compactSegment.getInterval()).build())) {
+      final Cursor cursor = cursorHolder.asCursor();
+      cursor.reset();
+      final ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
+      Assert.assertEquals(ColumnType.STRING, factory.getColumnCapabilities("ts").toColumnType());
+      Assert.assertEquals(ColumnType.STRING, factory.getColumnCapabilities("dim").toColumnType());
+      Assert.assertEquals(ColumnType.LONG, factory.getColumnCapabilities("x").toColumnType());
+      Assert.assertEquals(ColumnType.LONG, factory.getColumnCapabilities("y").toColumnType());
+      while (!cursor.isDone()) {
+        final ColumnValueSelector<?> selector1 = factory.makeColumnValueSelector("ts");
+        final DimensionSelector selector2 = factory.makeDimensionSelector(new DefaultDimensionSpec("dim", "dim"));
+        final DimensionSelector selector3 = factory.makeDimensionSelector(new DefaultDimensionSpec("x", "x"));
+        final DimensionSelector selector4 = factory.makeDimensionSelector(new DefaultDimensionSpec("y", "y"));
+        final DimensionSelector selector5 = factory.makeDimensionSelector(new DefaultDimensionSpec("val", "val"));
+
+        rowsFromSegment.add(
+            StringUtils.format(
+                "%s,%s,%s,%s,%s",
+                selector1.getObject(),
+                selector2.getObject(),
+                selector3.getObject(),
+                selector4.getObject(),
+                selector5.getObject()
+            )
+        );
+
+        cursor.advance();
+      }
+    }
+
+    Assert.assertEquals(
+        ImmutableList.of(
+            "2014-01-01T00:00:10Z,a,10,100,1",
+            "2014-01-01T00:01:20Z,a,10,100,1",
+            "2014-01-01T00:00:10Z,b,20,110,2",
+            "2014-01-01T00:01:20Z,b,20,110,2",
+            "2014-01-01T00:00:10Z,c,30,120,3",
+            "2014-01-01T00:01:20Z,c,30,120,3"
+        ),
+        rowsFromSegment
+    );
+  }
+
   private Pair<TaskStatus, DataSegmentsWithSchemas> runIndexTask() throws Exception
   {
     return runIndexTask(null, null, false);
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
index 3c144929546..84a1457ad91 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
@@ -1874,6 +1874,7 @@ public class CompactionTaskTest
                     aggregatorFactories.toArray(new AggregatorFactory[0]),
                     null,
                     null,
+                    null,
                     null
                 );
               }
diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionsSpec.java b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionsSpec.java
index 553d5172b5f..e529aa2d218 100644
--- a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionsSpec.java
+++ b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionsSpec.java
@@ -21,6 +21,7 @@ package org.apache.druid.data.input.impl;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
@@ -29,7 +30,9 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.apache.druid.guice.annotations.PublicApi;
+import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.parsers.ParserUtils;
+import org.apache.druid.segment.column.ColumnHolder;
 
 import javax.annotation.Nullable;
 import java.util.ArrayList;
@@ -44,14 +47,34 @@ import java.util.stream.Collectors;
 @PublicApi
 public class DimensionsSpec
 {
+  /**
+   * Parameter name for allowing any sort order. Also used as an MSQ context parameter, for some consistency between
+   * MSQ and native ingest configuration.
+   */
+  public static final String PARAMETER_FORCE_TIME_SORT = "forceSegmentSortByTime";
+
+  /**
+   * Warning about non-time ordering to include in error messages when {@link #PARAMETER_FORCE_TIME_SORT} is
+   * not set.
+   */
+  public static final String WARNING_NON_TIME_SORT_ORDER = StringUtils.format(
+      "Warning: support for segments not sorted by[%s] is experimental. Such segments are not readable by older "
+      + "version of Druid, and certain queries cannot run on them. See "
+      + "https://druid.apache.org/docs/latest/ingestion/partitioning#sorting for details before using this option.",
+      ColumnHolder.TIME_COLUMN_NAME
+  );
+
+  public static final boolean DEFAULT_FORCE_TIME_SORT = true;
+
   private final List<DimensionSchema> dimensions;
   private final Set<String> dimensionExclusions;
   private final Map<String, DimensionSchema> dimensionSchemaMap;
   private final boolean includeAllDimensions;
+  private final Boolean forceSegmentSortByTime;
 
   private final boolean useSchemaDiscovery;
 
-  public static final DimensionsSpec EMPTY = new DimensionsSpec(null, null, null, false, null);
+  public static final DimensionsSpec EMPTY = new DimensionsSpec(null, null, null, false, null, false);
 
   public static List<DimensionSchema> getDefaultSchemas(List<String> dimNames)
   {
@@ -80,7 +103,7 @@ public class DimensionsSpec
 
   public DimensionsSpec(List<DimensionSchema> dimensions)
   {
-    this(dimensions, null, null, false, null);
+    this(dimensions, null, null, false, null, null);
   }
 
   @JsonCreator
@@ -89,7 +112,8 @@ public class DimensionsSpec
       @JsonProperty("dimensionExclusions") List<String> dimensionExclusions,
       @Deprecated @JsonProperty("spatialDimensions") List<SpatialDimensionSchema> spatialDimensions,
       @JsonProperty("includeAllDimensions") boolean includeAllDimensions,
-      @JsonProperty("useSchemaDiscovery") Boolean useSchemaDiscovery
+      @JsonProperty("useSchemaDiscovery") Boolean useSchemaDiscovery,
+      @JsonProperty(PARAMETER_FORCE_TIME_SORT) Boolean forceSegmentSortByTime
   )
   {
     this.dimensions = dimensions == null
@@ -120,6 +144,7 @@ public class DimensionsSpec
     this.includeAllDimensions = includeAllDimensions;
     this.useSchemaDiscovery =
         useSchemaDiscovery != null && useSchemaDiscovery;
+    this.forceSegmentSortByTime = forceSegmentSortByTime;
   }
 
   @JsonProperty
@@ -146,6 +171,26 @@ public class DimensionsSpec
     return useSchemaDiscovery;
   }
 
+  @JsonProperty(PARAMETER_FORCE_TIME_SORT)
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  public Boolean isForceSegmentSortByTimeConfigured()
+  {
+    return forceSegmentSortByTime;
+  }
+
+  /**
+   * Returns {@link #isForceSegmentSortByTimeConfigured()} if nonnull, otherwise
+   * {@link #DEFAULT_FORCE_TIME_SORT}.
+   */
+  public boolean isForceSegmentSortByTime()
+  {
+    if (forceSegmentSortByTime != null) {
+      return forceSegmentSortByTime;
+    } else {
+      return DEFAULT_FORCE_TIME_SORT;
+    }
+  }
+
   @Deprecated
   @JsonIgnore
   public List<SpatialDimensionSchema> getSpatialDimensions()
@@ -191,9 +236,13 @@ public class DimensionsSpec
     return dimensionSchemaMap.get(dimension);
   }
 
-  public boolean hasCustomDimensions()
+  /**
+   * Whether this spec represents a set of fixed dimensions. Will be false if schema discovery is enabled, even if
+   * some dimensions are explicitly defined.
+   */
+  public boolean hasFixedDimensions()
   {
-    return !(dimensions == null || dimensions.isEmpty());
+    return dimensions != null && !dimensions.isEmpty() && !useSchemaDiscovery && !includeAllDimensions;
   }
 
   @PublicApi
@@ -204,7 +253,8 @@ public class DimensionsSpec
         ImmutableList.copyOf(dimensionExclusions),
         null,
         includeAllDimensions,
-        useSchemaDiscovery
+        useSchemaDiscovery,
+        forceSegmentSortByTime
     );
   }
 
@@ -215,7 +265,8 @@ public class DimensionsSpec
         ImmutableList.copyOf(Sets.union(dimensionExclusions, dimExs)),
         null,
         includeAllDimensions,
-        useSchemaDiscovery
+        useSchemaDiscovery,
+        forceSegmentSortByTime
     );
   }
 
@@ -227,7 +278,8 @@ public class DimensionsSpec
         ImmutableList.copyOf(dimensionExclusions),
         spatials,
         includeAllDimensions,
-        useSchemaDiscovery
+        useSchemaDiscovery,
+        forceSegmentSortByTime
     );
   }
 
@@ -269,7 +321,9 @@ public class DimensionsSpec
     return includeAllDimensions == that.includeAllDimensions
            && useSchemaDiscovery == that.useSchemaDiscovery
            && Objects.equals(dimensions, that.dimensions)
-           && Objects.equals(dimensionExclusions, that.dimensionExclusions);
+           && Objects.equals(dimensionExclusions, that.dimensionExclusions)
+           && Objects.equals(dimensionSchemaMap, that.dimensionSchemaMap)
+           && Objects.equals(forceSegmentSortByTime, that.forceSegmentSortByTime);
   }
 
   @Override
@@ -278,7 +332,9 @@ public class DimensionsSpec
     return Objects.hash(
         dimensions,
         dimensionExclusions,
+        dimensionSchemaMap,
         includeAllDimensions,
+        forceSegmentSortByTime,
         useSchemaDiscovery
     );
   }
@@ -291,6 +347,9 @@ public class DimensionsSpec
            ", dimensionExclusions=" + dimensionExclusions +
            ", includeAllDimensions=" + includeAllDimensions +
            ", useSchemaDiscovery=" + useSchemaDiscovery +
+           (forceSegmentSortByTime != null
+            ? ", forceSegmentSortByTime=" + forceSegmentSortByTime
+            : "") +
            '}';
   }
 
@@ -301,6 +360,7 @@ public class DimensionsSpec
     private List<SpatialDimensionSchema> spatialDimensions;
     private boolean includeAllDimensions;
     private boolean useSchemaDiscovery;
+    private Boolean forceSegmentSortByTime;
 
     public Builder setDimensions(List<DimensionSchema> dimensions)
     {
@@ -339,6 +399,12 @@ public class DimensionsSpec
       return this;
     }
 
+    public Builder setForceSegmentSortByTime(Boolean forceSegmentSortByTime)
+    {
+      this.forceSegmentSortByTime = forceSegmentSortByTime;
+      return this;
+    }
+
     public DimensionsSpec build()
     {
       return new DimensionsSpec(
@@ -346,7 +412,8 @@ public class DimensionsSpec
           dimensionExclusions,
           spatialDimensions,
           includeAllDimensions,
-          useSchemaDiscovery
+          useSchemaDiscovery,
+          forceSegmentSortByTime
       );
     }
   }
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/FrameStorageAdapter.java b/processing/src/main/java/org/apache/druid/frame/segment/FrameStorageAdapter.java
index 79dcdb7ced3..51ef56928d4 100644
--- a/processing/src/main/java/org/apache/druid/frame/segment/FrameStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/frame/segment/FrameStorageAdapter.java
@@ -31,7 +31,6 @@ import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.ListIndexed;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -87,18 +86,6 @@ public class FrameStorageAdapter implements StorageAdapter
     return DimensionDictionarySelector.CARDINALITY_UNKNOWN;
   }
 
-  @Override
-  public DateTime getMinTime()
-  {
-    return getInterval().getStart();
-  }
-
-  @Override
-  public DateTime getMaxTime()
-  {
-    return getInterval().getEnd().minus(1);
-  }
-
   @Nullable
   @Override
   public Comparable getMinValue(String column)
@@ -128,12 +115,6 @@ public class FrameStorageAdapter implements StorageAdapter
     return frame.numRows();
   }
 
-  @Override
-  public DateTime getMaxIngestedEventTime()
-  {
-    return getMaxTime();
-  }
-
   @Override
   @Nullable
   public Metadata getMetadata()
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorHolderFactory.java b/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorHolderFactory.java
index 7c97530e37f..ee1a01f17fe 100644
--- a/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorHolderFactory.java
+++ b/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorHolderFactory.java
@@ -28,6 +28,7 @@ import org.apache.druid.frame.segment.FrameFilteredOffset;
 import org.apache.druid.frame.segment.row.RowFrameCursorHolderFactory;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.vector.VectorValueMatcher;
@@ -36,10 +37,8 @@ import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.CursorHolderFactory;
-import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.QueryableIndexColumnSelectorFactory;
 import org.apache.druid.segment.SimpleAscendingOffset;
-import org.apache.druid.segment.SimpleDescendingOffset;
 import org.apache.druid.segment.SimpleSettableOffset;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.vector.FilteredVectorOffset;
@@ -51,6 +50,7 @@ import org.apache.druid.segment.vector.VectorOffset;
 import org.apache.druid.utils.CloseableUtils;
 
 import javax.annotation.Nullable;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -81,24 +81,19 @@ public class ColumnarFrameCursorHolderFactory implements CursorHolderFactory
   public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
     final Closer closer = Closer.create();
-    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
-    final List<OrderBy> ordering;
-    final boolean descending;
-    if (Cursors.preferDescendingTimeOrdering(spec)) {
-      ordering = Cursors.descendingTimeOrder();
-      descending = true;
-    } else {
-      ordering = Cursors.ascendingTimeOrder();
-      descending = false;
-    }
+
+    // Frames are not self-describing as to their sort order, so we can't determine the sort order by looking at
+    // the Frame object. We could populate this with information from the relevant ClusterBy, but that's not available
+    // at this point in the code. It could be plumbed in at some point. For now, use an empty list.
+    final List<OrderBy> ordering = Collections.emptyList();
+
     return new CursorHolder()
     {
       @Override
       public boolean canVectorize()
       {
         return (spec.getFilter() == null || spec.getFilter().canVectorizeMatcher(signature))
-               && spec.getVirtualColumns().canVectorize(signature)
-               && !descending;
+               && spec.getVirtualColumns().canVectorize(signature);
       }
 
       @Override
@@ -107,16 +102,13 @@ public class ColumnarFrameCursorHolderFactory implements CursorHolderFactory
         final FrameQueryableIndex index = new FrameQueryableIndex(frame, signature, columnReaders);
         final ColumnCache columnCache = new ColumnCache(index, closer);
         final Filter filterToUse = FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
-        final boolean descendingTimeOrder = Cursors.preferDescendingTimeOrdering(spec);
-        final SimpleSettableOffset baseOffset = descendingTimeOrder
-                                                ? new SimpleDescendingOffset(frame.numRows())
-                                                : new SimpleAscendingOffset(frame.numRows());
+        final SimpleSettableOffset baseOffset = new SimpleAscendingOffset(frame.numRows());
 
         final QueryableIndexColumnSelectorFactory columnSelectorFactory = new QueryableIndexColumnSelectorFactory(
-                spec.getVirtualColumns(),
-                descendingTimeOrder,
-                baseOffset,
-                columnCache
+            spec.getVirtualColumns(),
+            Order.NONE,
+            baseOffset,
+            columnCache
         );
 
         final SimpleSettableOffset offset;
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameQueryableIndex.java b/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameQueryableIndex.java
index 86bf7eb5126..d3a03ca83bb 100644
--- a/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameQueryableIndex.java
+++ b/processing/src/main/java/org/apache/druid/frame/segment/columnar/FrameQueryableIndex.java
@@ -24,6 +24,7 @@ import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
 import org.apache.druid.frame.Frame;
 import org.apache.druid.frame.FrameType;
 import org.apache.druid.frame.read.columnar.FrameColumnReader;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.DimensionHandler;
 import org.apache.druid.segment.Metadata;
 import org.apache.druid.segment.QueryableIndex;
@@ -100,6 +101,12 @@ public class FrameQueryableIndex implements QueryableIndex
     return new ListIndexed<>(signature.getColumnNames());
   }
 
+  @Override
+  public List<OrderBy> getOrdering()
+  {
+    return Collections.emptyList();
+  }
+
   @Override
   public BitmapFactory getBitmapFactoryForDimensions()
   {
diff --git a/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorHolderFactory.java b/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorHolderFactory.java
index 2d0d6567d5f..f84eb62a4fa 100644
--- a/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorHolderFactory.java
+++ b/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorHolderFactory.java
@@ -34,12 +34,11 @@ import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.CursorHolderFactory;
-import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.SimpleAscendingOffset;
-import org.apache.druid.segment.SimpleDescendingOffset;
 import org.apache.druid.segment.SimpleSettableOffset;
 
 import javax.annotation.Nullable;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -69,16 +68,11 @@ public class RowFrameCursorHolderFactory implements CursorHolderFactory
   @Override
   public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
-    final List<OrderBy> ordering;
-    final boolean descending;
-    if (Cursors.preferDescendingTimeOrdering(spec)) {
-      ordering = Cursors.descendingTimeOrder();
-      descending = true;
-    } else {
-      ordering = Cursors.ascendingTimeOrder();
-      descending = false;
-    }
+    // Frames are not self-describing as to their sort order, so we can't determine the sort order by looking at
+    // the Frame object. We could populate this with information from the relevant ClusterBy, but that's not available
+    // at this point in the code. It could be plumbed in at some point. For now, use an empty list.
+    final List<OrderBy> ordering = Collections.emptyList();
+
     return new CursorHolder()
     {
       @Nullable
@@ -87,10 +81,7 @@ public class RowFrameCursorHolderFactory implements CursorHolderFactory
       {
         final Filter filterToUse = FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
 
-        final SimpleSettableOffset baseOffset = descending
-                                                ? new SimpleDescendingOffset(frame.numRows())
-                                                : new SimpleAscendingOffset(frame.numRows());
-
+        final SimpleSettableOffset baseOffset = new SimpleAscendingOffset(frame.numRows());
 
         final ColumnSelectorFactory columnSelectorFactory =
             spec.getVirtualColumns().wrap(
diff --git a/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java b/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
index 3c8c89b2c26..05150ea830f 100644
--- a/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
+++ b/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
@@ -69,7 +69,6 @@ public class Granularities
     return granularity == null ? Granularities.ALL : granularity;
   }
 
-
   /**
    * Decorates {@link CursorBuildSpec} with a grouping column and virtual column equivalent to the {@link Granularity}
    * for a {@link Query}, if that query has granularity other than {@link Granularities#ALL}. If the query has 'ALL'
diff --git a/processing/src/main/java/org/apache/druid/query/CursorGranularizer.java b/processing/src/main/java/org/apache/druid/query/CursorGranularizer.java
index b9d3418779a..fa2defce7f0 100644
--- a/processing/src/main/java/org/apache/druid/query/CursorGranularizer.java
+++ b/processing/src/main/java/org/apache/druid/query/CursorGranularizer.java
@@ -22,11 +22,13 @@ package org.apache.druid.query;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
@@ -45,25 +47,39 @@ public class CursorGranularizer
 {
   @Nullable
   public static CursorGranularizer create(
-      final StorageAdapter storageAdapter,
       final Cursor cursor,
+      @Nullable final TimeBoundaryInspector timeBoundaryInspector,
+      final Order timeOrder,
       final Granularity granularity,
-      final Interval queryInterval,
-      final boolean descending
+      final Interval queryInterval
   )
   {
-    final DateTime minTime = storageAdapter.getMinTime();
-    final DateTime maxTime = storageAdapter.getMaxTime();
+    if (!Granularities.ALL.equals(granularity) && timeOrder == Order.NONE) {
+      throw DruidException
+          .forPersona(DruidException.Persona.USER)
+          .ofCategory(DruidException.Category.UNSUPPORTED)
+          .build("Cannot use granularity[%s] on non-time-sorted data.", granularity);
+    }
 
-    final Interval storageAdapterInterval = new Interval(minTime, granularity.bucketEnd(maxTime));
-    final Interval clippedQueryInterval = queryInterval.overlap(storageAdapterInterval);
+    final Interval clippedQueryInterval;
+
+    if (timeBoundaryInspector != null) {
+      clippedQueryInterval = queryInterval.overlap(
+          new Interval(
+              timeBoundaryInspector.getMinTime(),
+              granularity.bucketEnd(timeBoundaryInspector.getMaxTime())
+          )
+      );
+    } else {
+      clippedQueryInterval = queryInterval;
+    }
 
     if (clippedQueryInterval == null) {
       return null;
     }
 
     Iterable<Interval> bucketIterable = granularity.getIterable(clippedQueryInterval);
-    if (descending) {
+    if (timeOrder == Order.DESCENDING) {
       bucketIterable = Lists.reverse(ImmutableList.copyOf(bucketIterable));
     }
     final Interval firstBucket = granularity.bucket(clippedQueryInterval.getStart());
@@ -78,7 +94,7 @@ public class CursorGranularizer
       timeSelector = cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME);
     }
 
-    return new CursorGranularizer(cursor, bucketIterable, timeSelector, descending);
+    return new CursorGranularizer(cursor, bucketIterable, timeSelector, timeOrder == Order.DESCENDING);
   }
 
   private final Cursor cursor;
diff --git a/processing/src/main/java/org/apache/druid/query/FluentQueryRunner.java b/processing/src/main/java/org/apache/druid/query/FluentQueryRunner.java
index e8875761ad4..dcef6fa0651 100644
--- a/processing/src/main/java/org/apache/druid/query/FluentQueryRunner.java
+++ b/processing/src/main/java/org/apache/druid/query/FluentQueryRunner.java
@@ -104,31 +104,4 @@ public class FluentQueryRunner<T> implements QueryRunner<T>
   {
     return from(mapFn.apply(baseRunner));
   }
-
-  /**
-   * Sets the toString of the QueryRunner.  This is used because QueryRunner objects are often used as parameters for
-   * tests and the toString() value of the QueryRunners are used for the name of the test.
-   *
-   * This method doesn't return a FluentQueryRunner because it breaks the fluency.
-   *
-   * @param toStringValue the value that the resulting QueryRunner should return from its toString method.
-   * @return a QueryRunner that will return toStringValue from its toString method
-   */
-  public QueryRunner<T> setToString(String toStringValue)
-  {
-    return new QueryRunner<T>()
-    {
-      @Override
-      public Sequence<T> run(QueryPlus<T> queryPlus, ResponseContext responseContext)
-      {
-        return FluentQueryRunner.this.run(queryPlus, responseContext);
-      }
-
-      @Override
-      public String toString()
-      {
-        return toStringValue;
-      }
-    };
-  }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQueryRunnerFactory.java
index 31751fb13f1..c24f6f4f90a 100644
--- a/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQueryRunnerFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQueryRunnerFactory.java
@@ -33,8 +33,9 @@ import org.apache.druid.query.QueryToolChest;
 import org.apache.druid.query.QueryWatcher;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.context.ResponseContext;
+import org.apache.druid.segment.MaxIngestedEventTimeInspector;
 import org.apache.druid.segment.Segment;
-import org.apache.druid.segment.StorageAdapter;
+import org.joda.time.Interval;
 
 import java.util.Iterator;
 
@@ -79,11 +80,13 @@ public class DataSourceMetadataQueryRunnerFactory
 
   private static class DataSourceMetadataQueryRunner implements QueryRunner<Result<DataSourceMetadataResultValue>>
   {
-    private final StorageAdapter adapter;
+    private final Interval segmentInterval;
+    private final MaxIngestedEventTimeInspector inspector;
 
     public DataSourceMetadataQueryRunner(Segment segment)
     {
-      this.adapter = segment.asStorageAdapter();
+      this.segmentInterval = segment.asStorageAdapter().getInterval();
+      this.inspector = segment.as(MaxIngestedEventTimeInspector.class);
     }
 
     @Override
@@ -105,15 +108,9 @@ public class DataSourceMetadataQueryRunnerFactory
             @Override
             public Iterator<Result<DataSourceMetadataResultValue>> make()
             {
-              if (adapter == null) {
-                throw new ISE(
-                    "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped."
-                );
-              }
-
               return legacyQuery.buildResult(
-                  adapter.getInterval().getStart(),
-                  adapter.getMaxIngestedEventTime()
+                  segmentInterval.getStart(),
+                  (inspector != null ? inspector.getMaxIngestedEventTime() : null)
               ).iterator();
             }
 
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryRunnerFactory.java
index adf43e8cc16..dcde45aeaae 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryRunnerFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryRunnerFactory.java
@@ -32,6 +32,9 @@ import org.apache.druid.query.QueryToolChest;
 import org.apache.druid.query.context.ResponseContext;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
+
+import javax.annotation.Nullable;
 
 /**
  *
@@ -86,11 +89,14 @@ public class GroupByQueryRunnerFactory implements QueryRunnerFactory<ResultRow,
   private static class GroupByQueryRunner implements QueryRunner<ResultRow>
   {
     private final StorageAdapter adapter;
+    @Nullable
+    private final TimeBoundaryInspector timeBoundaryInspector;
     private final GroupingEngine groupingEngine;
 
     public GroupByQueryRunner(Segment segment, final GroupingEngine groupingEngine)
     {
       this.adapter = segment.asStorageAdapter();
+      this.timeBoundaryInspector = segment.as(TimeBoundaryInspector.class);
       this.groupingEngine = groupingEngine;
     }
 
@@ -102,7 +108,12 @@ public class GroupByQueryRunnerFactory implements QueryRunnerFactory<ResultRow,
         throw new ISE("Got a [%s] which isn't a %s", query.getClass(), GroupByQuery.class);
       }
 
-      return groupingEngine.process((GroupByQuery) query, adapter, (GroupByQueryMetrics) queryPlus.getQueryMetrics());
+      return groupingEngine.process(
+          (GroupByQuery) query,
+          adapter,
+          timeBoundaryInspector,
+          (GroupByQueryMetrics) queryPlus.getQueryMetrics()
+      );
     }
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupingEngine.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupingEngine.java
index 75d7ad50d8a..12b186afc72 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/GroupingEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupingEngine.java
@@ -80,6 +80,7 @@ import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionHandlerUtils;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnType;
@@ -424,7 +425,7 @@ public class GroupingEngine
   /**
    * Merges a variety of single-segment query runners into a combined runner. Used by
    * {@link GroupByQueryRunnerFactory#mergeRunners(QueryProcessingPool, Iterable)}. In
-   * that sense, it is intended to go along with {@link #process(GroupByQuery, StorageAdapter, GroupByQueryMetrics)} (the runners created
+   * that sense, it is intended to go along with {@link #process} (the runners created
    * by that method will be fed into this method).
    *
    * This is primarily called on the data servers, to merge the results from processing on the segments. This method can
@@ -464,14 +465,16 @@ public class GroupingEngine
    *
    * This method is only called on data servers, like Historicals (not the Broker).
    *
-   * @param query          the groupBy query
-   * @param storageAdapter storage adatper for the segment in question
+   * @param query                 the groupBy query
+   * @param storageAdapter        storage adatper for the segment in question
+   * @param timeBoundaryInspector time boundary inspector for the segment in question
    *
    * @return result sequence for the storage adapter
    */
   public Sequence<ResultRow> process(
       GroupByQuery query,
       StorageAdapter storageAdapter,
+      @Nullable TimeBoundaryInspector timeBoundaryInspector,
       @Nullable GroupByQueryMetrics groupByQueryMetrics
   )
   {
@@ -515,6 +518,7 @@ public class GroupingEngine
         result = VectorGroupByEngine.process(
             query,
             storageAdapter,
+            timeBoundaryInspector,
             cursorHolder,
             bufferHolder.get(),
             fudgeTimestamp,
@@ -526,6 +530,7 @@ public class GroupingEngine
         result = GroupByQueryEngine.process(
             query,
             storageAdapter,
+            timeBoundaryInspector,
             cursorHolder,
             buildSpec,
             bufferHolder.get(),
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByQueryEngine.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByQueryEngine.java
index 9f250772462..c6a32d36416 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByQueryEngine.java
@@ -39,7 +39,6 @@ import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
-import org.apache.druid.query.groupby.GroupByQueryMetrics;
 import org.apache.druid.query.groupby.GroupingEngine;
 import org.apache.druid.query.groupby.ResultRow;
 import org.apache.druid.query.groupby.epinephelinae.column.GroupByColumnSelectorPlus;
@@ -55,6 +54,7 @@ import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionHandlerUtils;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ValueType;
@@ -77,8 +77,7 @@ import java.util.stream.Stream;
  * <p>
  * This code runs on anything that processes {@link StorageAdapter} directly, typically data servers like Historicals.
  * <p>
- * Used for non-vectorized processing by
- * {@link GroupingEngine#process(GroupByQuery, StorageAdapter, GroupByQueryMetrics)}.
+ * Used for non-vectorized processing by {@link GroupingEngine#process}.
  *
  * This code runs on data servers, like Historicals and Peons, and also Brokers, if they operate on local datasources like
  * inlined data wherein the broker needs to process some portion of data like the data server
@@ -97,6 +96,7 @@ public class GroupByQueryEngine
   public static Sequence<ResultRow> process(
       final GroupByQuery query,
       final StorageAdapter storageAdapter,
+      @Nullable final TimeBoundaryInspector timeBoundaryInspector,
       final CursorHolder cursorHolder,
       final CursorBuildSpec buildSpec,
       final ByteBuffer processingBuffer,
@@ -110,11 +110,11 @@ public class GroupByQueryEngine
       return Sequences.empty();
     }
     final CursorGranularizer granularizer = CursorGranularizer.create(
-        storageAdapter,
         cursor,
+        timeBoundaryInspector,
+        cursorHolder.getTimeOrder(),
         query.getGranularity(),
-        buildSpec.getInterval(),
-        false
+        buildSpec.getInterval()
     );
     if (granularizer == null) {
       return Sequences.empty();
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
index 4d0a30547c2..c6f52e0bd21 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
@@ -27,12 +27,12 @@ import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.query.DruidProcessingConfig;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.aggregation.AggregatorAdapters;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
-import org.apache.druid.query.groupby.GroupByQueryMetrics;
 import org.apache.druid.query.groupby.GroupingEngine;
 import org.apache.druid.query.groupby.ResultRow;
 import org.apache.druid.query.groupby.epinephelinae.AggregateResult;
@@ -46,6 +46,7 @@ import org.apache.druid.segment.ColumnInspector;
 import org.apache.druid.segment.ColumnProcessors;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
 import org.apache.druid.segment.vector.VectorCursor;
@@ -65,8 +66,7 @@ import java.util.stream.Collectors;
  * Contains logic to process a groupBy query on a single {@link StorageAdapter} in a vectorized manner.
  * This code runs on anything that processes {@link StorageAdapter} directly, typically data servers like Historicals.
  * <p>
- * Used for vectorized processing by
- * {@link GroupingEngine#process(GroupByQuery, StorageAdapter, GroupByQueryMetrics)}.
+ * Used for vectorized processing by {@link GroupingEngine#process}.
  *
  * @see org.apache.druid.query.groupby.epinephelinae.GroupByQueryEngine for non-vectorized version of this logic
  */
@@ -80,6 +80,7 @@ public class VectorGroupByEngine
   public static Sequence<ResultRow> process(
       final GroupByQuery query,
       final StorageAdapter storageAdapter,
+      @Nullable TimeBoundaryInspector timeBoundaryInspector,
       final CursorHolder cursorHolder,
       final ByteBuffer processingBuffer,
       @Nullable final DateTime fudgeTimestamp,
@@ -145,7 +146,9 @@ public class VectorGroupByEngine
                 config,
                 processingConfig,
                 storageAdapter,
+                timeBoundaryInspector,
                 cursor,
+                cursorHolder.getTimeOrder(),
                 interval,
                 dimensions,
                 processingBuffer,
@@ -216,7 +219,7 @@ public class VectorGroupByEngine
     private final VectorGrouper vectorGrouper;
 
     @Nullable
-    private final VectorCursorGranularizer granulizer;
+    private final VectorCursorGranularizer granularizer;
 
     // Granularity-bucket iterator and current bucket.
     private final Iterator<Interval> bucketIterator;
@@ -239,7 +242,9 @@ public class VectorGroupByEngine
         final GroupByQueryConfig querySpecificConfig,
         final DruidProcessingConfig processingConfig,
         final StorageAdapter storageAdapter,
+        @Nullable TimeBoundaryInspector timeBoundaryInspector,
         final VectorCursor cursor,
+        final Order timeOrder,
         final Interval queryInterval,
         final List<GroupByVectorColumnSelector> selectors,
         final ByteBuffer processingBuffer,
@@ -257,10 +262,16 @@ public class VectorGroupByEngine
       this.keySize = selectors.stream().mapToInt(GroupByVectorColumnSelector::getGroupingKeySize).sum();
       this.keySpace = WritableMemory.allocate(keySize * cursor.getMaxVectorSize());
       this.vectorGrouper = makeGrouper();
-      this.granulizer = VectorCursorGranularizer.create(storageAdapter, cursor, query.getGranularity(), queryInterval);
+      this.granularizer = VectorCursorGranularizer.create(
+          cursor,
+          timeBoundaryInspector,
+          timeOrder,
+          query.getGranularity(),
+          queryInterval
+      );
 
-      if (granulizer != null) {
-        this.bucketIterator = granulizer.getBucketIterable().iterator();
+      if (granularizer != null) {
+        this.bucketIterator = granularizer.getBucketIterable().iterator();
       } else {
         this.bucketIterator = Collections.emptyIterator();
       }
@@ -368,20 +379,20 @@ public class VectorGroupByEngine
         final int startOffset;
 
         if (partiallyAggregatedRows < 0) {
-          granulizer.setCurrentOffsets(bucketInterval);
-          startOffset = granulizer.getStartOffset();
+          granularizer.setCurrentOffsets(bucketInterval);
+          startOffset = granularizer.getStartOffset();
         } else {
-          startOffset = granulizer.getStartOffset() + partiallyAggregatedRows;
+          startOffset = granularizer.getStartOffset() + partiallyAggregatedRows;
         }
 
-        if (granulizer.getEndOffset() > startOffset) {
+        if (granularizer.getEndOffset() > startOffset) {
           // Write keys to the keySpace.
           int keyOffset = 0;
           for (final GroupByVectorColumnSelector selector : selectors) {
             // Update selectorInternalFootprint now, but check it later. (We reset on the first vector that causes us
             // to go past the limit.)
             selectorInternalFootprint +=
-                selector.writeKeys(keySpace, keySize, keyOffset, startOffset, granulizer.getEndOffset());
+                selector.writeKeys(keySpace, keySize, keyOffset, startOffset, granularizer.getEndOffset());
 
             keyOffset += selector.getGroupingKeySize();
           }
@@ -390,7 +401,7 @@ public class VectorGroupByEngine
           final AggregateResult result = vectorGrouper.aggregateVector(
               keySpace,
               startOffset,
-              granulizer.getEndOffset()
+              granularizer.getEndOffset()
           );
 
           if (result.isOk()) {
@@ -408,7 +419,7 @@ public class VectorGroupByEngine
 
         if (partiallyAggregatedRows >= 0) {
           break;
-        } else if (!granulizer.advanceCursorWithinBucket()) {
+        } else if (!granularizer.advanceCursorWithinBucket()) {
           // Advance bucketInterval.
           bucketInterval = bucketIterator.hasNext() ? bucketIterator.next() : null;
           break;
diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
index 893480f5453..ca2c2725c6f 100644
--- a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.UOE;
@@ -119,8 +120,20 @@ public class ScanQueryEngine
     responseContext.addRowScanCount(0);
     final long limit = calculateRemainingScanRowsLimit(query, responseContext);
     final CursorHolder cursorHolder = adapter.makeCursorHolder(makeCursorBuildSpec(query, queryMetrics));
-    if (Order.NONE != query.getTimeOrder()) {
-      Cursors.requireTimeOrdering(cursorHolder, query.getTimeOrder());
+    if (Order.NONE != query.getTimeOrder()
+        && Cursors.getTimeOrdering(cursorHolder.getOrdering()) != query.getTimeOrder()) {
+      final String failureReason = StringUtils.format(
+          "Cannot order by[%s] with direction[%s] on cursor with order[%s].",
+          ColumnHolder.TIME_COLUMN_NAME,
+          query.getTimeOrder(),
+          cursorHolder.getOrdering()
+      );
+
+      cursorHolder.close();
+
+      throw DruidException.forPersona(DruidException.Persona.USER)
+                          .ofCategory(DruidException.Category.UNSUPPORTED)
+                          .build("%s", failureReason);
     }
     return new BaseSequence<>(
         new BaseSequence.IteratorMaker<ScanResultValue, Iterator<ScanResultValue>>()
diff --git a/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
index c522e70c90b..d5af263a1c7 100644
--- a/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
+++ b/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
@@ -27,6 +27,7 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap;
 import org.apache.druid.collections.bitmap.MutableBitmap;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.query.DefaultBitmapResultFactory;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.extraction.ExtractionFn;
 import org.apache.druid.query.extraction.IdentityExtractionFn;
@@ -34,6 +35,7 @@ import org.apache.druid.query.filter.ColumnIndexSelector;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.search.CursorOnlyStrategy.CursorBasedExecutor;
 import org.apache.druid.segment.ColumnSelectorColumnIndexSelector;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.DeprecatedQueryableIndexColumnSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.Segment;
@@ -98,7 +100,8 @@ public class UseIndexesStrategy extends SearchStrategy
         // Note: if some filters support bitmap indexes but others are not, the current implementation always employs
         // the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index
         // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps.
-        if (filter == null || filter.getBitmapColumnIndex(selector) != null) {
+        if ((filter == null || filter.getBitmapColumnIndex(selector) != null)
+            && Cursors.getTimeOrdering(index.getOrdering()) == Order.ASCENDING) {
           final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(
               index,
               segment,
diff --git a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java
index 17db30455e0..47412f7d7d5 100644
--- a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java
@@ -200,6 +200,16 @@ public class TimeBoundaryQuery extends BaseQuery<Result<TimeBoundaryResultValue>
     return bound.equalsIgnoreCase(MAX_TIME);
   }
 
+  boolean needsMinTime()
+  {
+    return !isMaxTime();
+  }
+
+  boolean needsMaxTime()
+  {
+    return !isMinTime();
+  }
+
   @Override
   public String toString()
   {
diff --git a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
index 38b5ff6eb93..ec9e27c6f38 100644
--- a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
@@ -19,8 +19,8 @@
 
 package org.apache.druid.query.timeboundary;
 
-import com.google.common.base.Function;
 import com.google.inject.Inject;
+import it.unimi.dsi.fastutil.Pair;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
@@ -36,17 +36,20 @@ import org.apache.druid.query.QueryRunnerFactory;
 import org.apache.druid.query.QueryToolChest;
 import org.apache.druid.query.QueryWatcher;
 import org.apache.druid.query.Result;
-import org.apache.druid.query.TableDataSource;
 import org.apache.druid.query.context.ResponseContext;
 import org.apache.druid.segment.BaseLongColumnValueSelector;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.CursorHolderFactory;
 import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.filter.Filters;
+import org.apache.druid.segment.vector.VectorCursor;
+import org.apache.druid.segment.vector.VectorValueSelector;
 import org.apache.druid.utils.CollectionUtils;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
@@ -93,47 +96,13 @@ public class TimeBoundaryQueryRunnerFactory
   private static class TimeBoundaryQueryRunner implements QueryRunner<Result<TimeBoundaryResultValue>>
   {
     private final StorageAdapter adapter;
-    private final Function<Cursor, Result<DateTime>> skipToFirstMatching;
+    @Nullable
+    private final TimeBoundaryInspector timeBoundaryInspector;
 
     public TimeBoundaryQueryRunner(Segment segment)
     {
       this.adapter = segment.asStorageAdapter();
-      this.skipToFirstMatching = new Function<Cursor, Result<DateTime>>()
-      {
-        @SuppressWarnings("ArgumentParameterSwap")
-        @Override
-        public Result<DateTime> apply(Cursor cursor)
-        {
-          if (cursor.isDone()) {
-            return null;
-          }
-          final BaseLongColumnValueSelector timestampColumnSelector =
-              cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME);
-          final DateTime timestamp = DateTimes.utc(timestampColumnSelector.getLong());
-          return new Result<>(adapter.getInterval().getStart(), timestamp);
-        }
-      };
-    }
-
-    @Nullable
-    private DateTime getTimeBoundary(StorageAdapter adapter, TimeBoundaryQuery legacyQuery, boolean descending)
-    {
-      final CursorBuildSpec.CursorBuildSpecBuilder bob = CursorBuildSpec.builder(makeCursorBuildSpec(legacyQuery));
-      if (descending) {
-        bob.setPreferredOrdering(Cursors.descendingTimeOrder());
-      } else {
-        bob.setPreferredOrdering(Cursors.ascendingTimeOrder());
-      }
-
-      try (final CursorHolder cursorHolder = adapter.makeCursorHolder(bob.build())) {
-        Cursors.requireTimeOrdering(cursorHolder, descending ? Order.DESCENDING : Order.ASCENDING);
-        final Cursor cursor = cursorHolder.asCursor();
-        if (cursor == null) {
-          return null;
-        }
-        final Result<DateTime> result = skipToFirstMatching.apply(cursor);
-        return result == null ? null : result.getValue();
-      }
+      this.timeBoundaryInspector = segment.as(TimeBoundaryInspector.class);
     }
 
     @Override
@@ -164,24 +133,18 @@ public class TimeBoundaryQueryRunnerFactory
               DateTime minTime = null;
               DateTime maxTime = null;
 
-              if (canUseAdapterMinMaxTime(query, adapter)) {
-                if (!query.isMaxTime()) {
-                  minTime = adapter.getMinTime();
+              if (canUseTimeBoundaryInspector(query, timeBoundaryInspector)) {
+                if (query.needsMinTime()) {
+                  minTime = timeBoundaryInspector.getMinTime();
                 }
 
-                if (!query.isMinTime()) {
-                  maxTime = adapter.getMaxTime();
+                if (query.needsMaxTime()) {
+                  maxTime = timeBoundaryInspector.getMaxTime();
                 }
               } else {
-                if (!query.isMaxTime()) {
-                  minTime = getTimeBoundary(adapter, query, false);
-                }
-
-                if (!query.isMinTime()) {
-                  if (query.isMaxTime() || minTime != null) {
-                    maxTime = getTimeBoundary(adapter, query, true);
-                  }
-                }
+                final Pair<DateTime, DateTime> timeBoundary = getTimeBoundary(query, adapter);
+                minTime = timeBoundary.left();
+                maxTime = timeBoundary.right();
               }
 
               return query.buildResult(
@@ -202,21 +165,22 @@ public class TimeBoundaryQueryRunnerFactory
   }
 
   /**
-   * Whether a particular {@link TimeBoundaryQuery} can use {@link StorageAdapter#getMinTime()} and/or
-   * {@link StorageAdapter#getMaxTime()}. If false, must use {@link StorageAdapter#makeCursorHolder(CursorBuildSpec)}.
+   * Whether a particular {@link TimeBoundaryQuery} can use {@link TimeBoundaryInspector#getMinTime()} and/or
+   * {@link TimeBoundaryInspector#getMaxTime()}.
+   *
+   * If false, must use {@link StorageAdapter#makeCursorHolder(CursorBuildSpec)}.
    */
-  private static boolean canUseAdapterMinMaxTime(final TimeBoundaryQuery query, final StorageAdapter adapter)
+  private static boolean canUseTimeBoundaryInspector(
+      final TimeBoundaryQuery query,
+      @Nullable final TimeBoundaryInspector timeBoundaryInspector
+  )
   {
-    if (query.getFilter() != null) {
-      // We have to check which rows actually match the filter.
+    if (timeBoundaryInspector == null || !timeBoundaryInspector.isMinMaxExact()) {
       return false;
     }
 
-    if (!(query.getDataSource() instanceof TableDataSource)) {
-      // In general, minTime / maxTime are only guaranteed to match data for regular tables.
-      //
-      // One example: an INNER JOIN can act as a filter and remove some rows. Another example: RowBasedStorageAdapter
-      // (used by e.g. inline data) uses nominal interval, not actual data, for minTime / maxTime.
+    if (query.getFilter() != null) {
+      // We have to check which rows actually match the filter.
       return false;
     }
 
@@ -225,8 +189,8 @@ public class TimeBoundaryQueryRunnerFactory
         xs -> new IAE("Should only have one interval, got[%s]", xs)
     );
 
-    if (!queryInterval.contains(adapter.getInterval())) {
-      // Query interval does not contain adapter interval. Need to create a cursor to see the first
+    if (!queryInterval.contains(timeBoundaryInspector.getMinMaxInterval())) {
+      // Query interval does not contain segment interval. Need to create a cursor to see the first
       // timestamp within the query interval.
       return false;
     }
@@ -244,4 +208,169 @@ public class TimeBoundaryQueryRunnerFactory
                           .setQueryContext(query.context())
                           .build();
   }
+
+  private static Pair<DateTime, DateTime> getTimeBoundary(
+      final TimeBoundaryQuery query,
+      final CursorHolderFactory cursorHolderFactory
+  )
+  {
+    DateTime minTime = null, maxTime = null;
+
+    if (query.needsMinTime()) {
+      final CursorBuildSpec cursorSpec =
+          CursorBuildSpec.builder(makeCursorBuildSpec(query))
+                         .setPreferredOrdering(Cursors.ascendingTimeOrder())
+                         .build();
+
+      try (final CursorHolder cursorHolder = cursorHolderFactory.makeCursorHolder(cursorSpec)) {
+        if (cursorHolder.getTimeOrder() == Order.ASCENDING) {
+          // Time-ordered cursor, use the first timestamp.
+          minTime = getFirstTimestamp(query, cursorHolder);
+        } else {
+          // Non-time-ordered cursor. Walk and find both minTime, maxTime. Return immediately.
+          return getTimeBoundaryFullScan(query, cursorHolder);
+        }
+      }
+    }
+
+    if (query.needsMaxTime()) {
+      final CursorBuildSpec cursorSpec =
+          CursorBuildSpec.builder(makeCursorBuildSpec(query))
+                         .setPreferredOrdering(Cursors.descendingTimeOrder())
+                         .build();
+
+      try (final CursorHolder cursorHolder = cursorHolderFactory.makeCursorHolder(cursorSpec)) {
+        if (cursorHolder.getTimeOrder() == Order.DESCENDING) {
+          // Time-ordered cursor, use the first timestamp.
+          maxTime = getFirstTimestamp(query, cursorHolder);
+        } else {
+          // Non-time-ordered cursor. Walk and find both minTime, maxTime. Return immediately.
+          return getTimeBoundaryFullScan(query, cursorHolder);
+        }
+      }
+    }
+
+    return Pair.of(minTime, maxTime);
+  }
+
+  /**
+   * Retrieve the first timestamp from {@link CursorHolder}.
+   */
+  @Nullable
+  private static DateTime getFirstTimestamp(
+      final TimeBoundaryQuery query,
+      final CursorHolder cursorHolder
+  )
+  {
+    if (query.context().getVectorize().shouldVectorize(cursorHolder.canVectorize())) {
+      return getFirstTimestampVectorized(cursorHolder.asVectorCursor());
+    } else {
+      return getFirstTimestampNonVectorized(cursorHolder.asCursor());
+    }
+  }
+
+  /**
+   * Retrieve the first timestamp from {@link CursorHolder#asVectorCursor()}.
+   */
+  @Nullable
+  private static DateTime getFirstTimestampVectorized(@Nullable final VectorCursor cursor)
+  {
+    if (cursor == null || cursor.isDone()) {
+      return null;
+    }
+    final VectorValueSelector timestampSelector =
+        cursor.getColumnSelectorFactory().makeValueSelector(ColumnHolder.TIME_COLUMN_NAME);
+    final long[] timeVector = timestampSelector.getLongVector();
+    return DateTimes.utc(timeVector[0]);
+  }
+
+  /**
+   * Retrieve the first timestamp from {@link CursorHolder#asCursor()}.
+   */
+  @Nullable
+  private static DateTime getFirstTimestampNonVectorized(@Nullable final Cursor cursor)
+  {
+    if (cursor == null || cursor.isDone()) {
+      return null;
+    }
+    final BaseLongColumnValueSelector timestampSelector =
+        cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME);
+    return DateTimes.utc(timestampSelector.getLong());
+  }
+
+  /**
+   * Retrieve min/max timestamps from {@link CursorHolder} using a full scan.
+   */
+  private static Pair<DateTime, DateTime> getTimeBoundaryFullScan(
+      final TimeBoundaryQuery query,
+      final CursorHolder cursorHolder
+  )
+  {
+    if (query.context().getVectorize().shouldVectorize(cursorHolder.canVectorize())) {
+      return getTimeBoundaryFullScanVectorized(query, cursorHolder.asVectorCursor());
+    } else {
+      return getTimeBoundaryFullScanNonVectorized(query, cursorHolder.asCursor());
+    }
+  }
+
+  /**
+   * Retrieve min/max timestamps from {@link CursorHolder#asVectorCursor()} using a full scan.
+   */
+  private static Pair<DateTime, DateTime> getTimeBoundaryFullScanVectorized(
+      final TimeBoundaryQuery query,
+      @Nullable final VectorCursor cursor
+  )
+  {
+    if (cursor == null || cursor.isDone()) {
+      return Pair.of(null, null);
+    }
+
+    final VectorValueSelector timeSelector =
+        cursor.getColumnSelectorFactory().makeValueSelector(ColumnHolder.TIME_COLUMN_NAME);
+
+    long minTime = Long.MAX_VALUE, maxTime = Long.MIN_VALUE;
+    while (!cursor.isDone()) {
+      final long[] timeVector = timeSelector.getLongVector();
+      for (int i = 0; i < cursor.getCurrentVectorSize(); i++) {
+        final long timestamp = timeVector[i];
+        minTime = Math.min(minTime, timestamp);
+        maxTime = Math.max(maxTime, timestamp);
+      }
+      cursor.advance();
+    }
+
+    return Pair.of(
+        query.needsMinTime() ? DateTimes.utc(minTime) : null,
+        query.needsMaxTime() ? DateTimes.utc(maxTime) : null
+    );
+  }
+
+  /**
+   * Retrieve min/max timestamps from {@link CursorHolder#asCursor()} using a full scan.
+   */
+  private static Pair<DateTime, DateTime> getTimeBoundaryFullScanNonVectorized(
+      final TimeBoundaryQuery query,
+      @Nullable final Cursor cursor
+  )
+  {
+    if (cursor == null || cursor.isDone()) {
+      return Pair.of(null, null);
+    }
+
+    final BaseLongColumnValueSelector timeSelector =
+        cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME);
+
+    long minTime = Long.MAX_VALUE, maxTime = Long.MIN_VALUE;
+    while (!cursor.isDone()) {
+      final long timestamp = timeSelector.getLong();
+      minTime = Math.min(minTime, timestamp);
+      maxTime = Math.max(maxTime, timestamp);
+      cursor.advance();
+    }
+
+    return Pair.of(
+        query.needsMinTime() ? DateTimes.utc(minTime) : null,
+        query.needsMaxTime() ? DateTimes.utc(maxTime) : null
+    );
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryEngine.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryEngine.java
index e23f8e19e6d..95cba2e647f 100644
--- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryEngine.java
@@ -33,7 +33,6 @@ import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.query.CursorGranularizer;
-import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.aggregation.Aggregator;
@@ -47,6 +46,7 @@ import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.SegmentMissingException;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
 import org.apache.druid.segment.vector.VectorCursor;
@@ -88,6 +88,7 @@ public class TimeseriesQueryEngine
   public Sequence<Result<TimeseriesResultValue>> process(
       final TimeseriesQuery query,
       final StorageAdapter adapter,
+      @Nullable TimeBoundaryInspector timeBoundaryInspector,
       @Nullable final TimeseriesQueryMetrics timeseriesQueryMetrics
   )
   {
@@ -100,16 +101,14 @@ public class TimeseriesQueryEngine
     final Interval interval = Iterables.getOnlyElement(query.getIntervals());
     final Granularity gran = query.getGranularity();
 
-
     final CursorHolder cursorHolder = adapter.makeCursorHolder(makeCursorBuildSpec(query, timeseriesQueryMetrics));
-    Cursors.requireTimeOrdering(cursorHolder, query.isDescending() ? Order.DESCENDING : Order.ASCENDING);
     try {
       final Sequence<Result<TimeseriesResultValue>> result;
 
       if (query.context().getVectorize().shouldVectorize(cursorHolder.canVectorize())) {
-        result = processVectorized(query, adapter, cursorHolder, interval, gran);
+        result = processVectorized(query, cursorHolder, timeBoundaryInspector, interval, gran);
       } else {
-        result = processNonVectorized(query, adapter, cursorHolder, interval, gran);
+        result = processNonVectorized(query, cursorHolder, timeBoundaryInspector, interval, gran);
       }
 
       final int limit = query.getLimit();
@@ -127,8 +126,8 @@ public class TimeseriesQueryEngine
 
   private Sequence<Result<TimeseriesResultValue>> processVectorized(
       final TimeseriesQuery query,
-      final StorageAdapter adapter,
       final CursorHolder cursorHolder,
+      @Nullable final TimeBoundaryInspector timeBoundaryInspector,
       final Interval queryInterval,
       final Granularity gran
   )
@@ -145,8 +144,9 @@ public class TimeseriesQueryEngine
     final Closer closer = Closer.create();
     try {
       final VectorCursorGranularizer granularizer = VectorCursorGranularizer.create(
-          adapter,
           cursor,
+          timeBoundaryInspector,
+          cursorHolder.getTimeOrder(),
           gran,
           queryInterval
       );
@@ -243,8 +243,8 @@ public class TimeseriesQueryEngine
 
   private Sequence<Result<TimeseriesResultValue>> processNonVectorized(
       final TimeseriesQuery query,
-      final StorageAdapter adapter,
       final CursorHolder cursorHolder,
+      @Nullable TimeBoundaryInspector timeBoundaryInspector,
       final Interval queryInterval,
       final Granularity gran
   )
@@ -256,11 +256,11 @@ public class TimeseriesQueryEngine
       return Sequences.empty();
     }
     final CursorGranularizer granularizer = CursorGranularizer.create(
-        adapter,
         cursor,
+        timeBoundaryInspector,
+        cursorHolder.getTimeOrder(),
         gran,
-        queryInterval,
-        query.isDescending()
+        queryInterval
     );
     if (granularizer == null) {
       return Sequences.empty();
diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerFactory.java
index fe3d420e566..788e72015e2 100644
--- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerFactory.java
@@ -34,6 +34,9 @@ import org.apache.druid.query.Result;
 import org.apache.druid.query.context.ResponseContext;
 import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
+
+import javax.annotation.Nullable;
 
 /**
  */
@@ -59,7 +62,7 @@ public class TimeseriesQueryRunnerFactory
   @Override
   public QueryRunner<Result<TimeseriesResultValue>> createRunner(final Segment segment)
   {
-    return new TimeseriesQueryRunner(engine, segment.asStorageAdapter());
+    return new TimeseriesQueryRunner(engine, segment.asStorageAdapter(), segment.as(TimeBoundaryInspector.class));
   }
 
   @Override
@@ -81,11 +84,18 @@ public class TimeseriesQueryRunnerFactory
   {
     private final TimeseriesQueryEngine engine;
     private final StorageAdapter adapter;
+    @Nullable
+    private final TimeBoundaryInspector timeBoundaryInspector;
 
-    private TimeseriesQueryRunner(TimeseriesQueryEngine engine, StorageAdapter adapter)
+    private TimeseriesQueryRunner(
+        TimeseriesQueryEngine engine,
+        StorageAdapter adapter,
+        @Nullable TimeBoundaryInspector timeBoundaryInspector
+    )
     {
       this.engine = engine;
       this.adapter = adapter;
+      this.timeBoundaryInspector = timeBoundaryInspector;
     }
 
     @Override
@@ -99,7 +109,12 @@ public class TimeseriesQueryRunnerFactory
         throw new ISE("Got a [%s] which isn't a %s", input.getClass(), TimeseriesQuery.class);
       }
 
-      return engine.process((TimeseriesQuery) input, adapter, (TimeseriesQueryMetrics) queryPlus.getQueryMetrics());
+      return engine.process(
+          (TimeseriesQuery) input,
+          adapter,
+          timeBoundaryInspector,
+          (TimeseriesQueryMetrics) queryPlus.getQueryMetrics()
+      );
     }
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryEngine.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryEngine.java
index ac130d3fa86..c6561faeee3 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryEngine.java
@@ -35,6 +35,7 @@ import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.SegmentMissingException;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.Types;
@@ -66,7 +67,8 @@ public class TopNQueryEngine
   public Sequence<Result<TopNResultValue>> query(
       final TopNQuery query,
       final StorageAdapter adapter,
-      final @Nullable TopNQueryMetrics queryMetrics
+      @Nullable final TimeBoundaryInspector timeBoundaryInspector,
+      @Nullable final TopNQueryMetrics queryMetrics
   )
   {
     if (adapter == null) {
@@ -84,11 +86,11 @@ public class TopNQueryEngine
       return Sequences.withBaggage(Sequences.empty(), cursorHolder);
     }
     final CursorGranularizer granularizer = CursorGranularizer.create(
-        adapter,
         cursor,
+        timeBoundaryInspector,
+        cursorHolder.getTimeOrder(),
         query.getGranularity(),
-        buildSpec.getInterval(),
-        false
+        buildSpec.getInterval()
     );
     if (granularizer == null) {
       return Sequences.withBaggage(Sequences.empty(), cursorHolder);
diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryRunnerFactory.java
index 821d37e3a19..a81d9258c86 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryRunnerFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryRunnerFactory.java
@@ -34,10 +34,12 @@ import org.apache.druid.query.QueryWatcher;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.context.ResponseContext;
 import org.apache.druid.segment.Segment;
+import org.apache.druid.segment.TimeBoundaryInspector;
 
 import java.nio.ByteBuffer;
 
 /**
+ *
  */
 public class TopNQueryRunnerFactory implements QueryRunnerFactory<Result<TopNResultValue>, TopNQuery>
 {
@@ -74,7 +76,12 @@ public class TopNQueryRunnerFactory implements QueryRunnerFactory<Result<TopNRes
         }
 
         TopNQuery query = (TopNQuery) input.getQuery();
-        return queryEngine.query(query, segment.asStorageAdapter(), (TopNQueryMetrics) input.getQueryMetrics());
+        return queryEngine.query(
+            query,
+            segment.asStorageAdapter(),
+            segment.as(TimeBoundaryInspector.class),
+            (TopNQueryMetrics) input.getQueryMetrics()
+        );
       }
     };
 
diff --git a/processing/src/main/java/org/apache/druid/query/vector/VectorCursorGranularizer.java b/processing/src/main/java/org/apache/druid/query/vector/VectorCursorGranularizer.java
index d899e1f0888..f262a4ea486 100644
--- a/processing/src/main/java/org/apache/druid/query/vector/VectorCursorGranularizer.java
+++ b/processing/src/main/java/org/apache/druid/query/vector/VectorCursorGranularizer.java
@@ -20,12 +20,14 @@
 package org.apache.druid.query.vector;
 
 import com.google.common.collect.Iterables;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.query.Order;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.vector.VectorCursor;
 import org.apache.druid.segment.vector.VectorValueSelector;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -42,17 +44,32 @@ public class VectorCursorGranularizer
 {
   @Nullable
   public static VectorCursorGranularizer create(
-      final StorageAdapter storageAdapter,
       final VectorCursor cursor,
+      @Nullable final TimeBoundaryInspector timeBoundaryInspector,
+      final Order timeOrder,
       final Granularity granularity,
       final Interval queryInterval
   )
   {
-    final DateTime minTime = storageAdapter.getMinTime();
-    final DateTime maxTime = storageAdapter.getMaxTime();
+    if (!Granularities.ALL.equals(granularity) && timeOrder == Order.NONE) {
+      throw DruidException
+          .forPersona(DruidException.Persona.USER)
+          .ofCategory(DruidException.Category.UNSUPPORTED)
+          .build("Cannot use granularity[%s] on non-time-sorted data.", granularity);
+    }
 
-    final Interval storageAdapterInterval = new Interval(minTime, granularity.bucketEnd(maxTime));
-    final Interval clippedQueryInterval = queryInterval.overlap(storageAdapterInterval);
+    final Interval clippedQueryInterval;
+
+    if (timeBoundaryInspector != null) {
+      clippedQueryInterval = queryInterval.overlap(
+          new Interval(
+              timeBoundaryInspector.getMinTime(),
+              granularity.bucketEnd(timeBoundaryInspector.getMaxTime())
+          )
+      );
+    } else {
+      clippedQueryInterval = queryInterval;
+    }
 
     if (clippedQueryInterval == null) {
       return null;
diff --git a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
index f6cbbdeffe8..a45c4b2d3e0 100644
--- a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
+++ b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.segment;
 
+import com.google.common.base.Preconditions;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.QueryContext;
@@ -28,6 +29,7 @@ import org.apache.druid.query.filter.Filter;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
+import java.util.Collections;
 import java.util.List;
 
 public class CursorBuildSpec
@@ -52,7 +54,6 @@ public class CursorBuildSpec
   private final VirtualColumns virtualColumns;
   @Nullable
   private final List<AggregatorFactory> aggregators;
-  @Nullable
   private final List<OrderBy> orderByColumns;
 
   private final QueryContext queryContext;
@@ -66,18 +67,18 @@ public class CursorBuildSpec
       @Nullable List<String> groupingColumns,
       VirtualColumns virtualColumns,
       @Nullable List<AggregatorFactory> aggregators,
-      @Nullable List<OrderBy> preferredOrdering,
+      List<OrderBy> preferredOrdering,
       QueryContext queryContext,
       @Nullable QueryMetrics<?> queryMetrics
   )
   {
     this.filter = filter;
-    this.interval = interval;
+    this.interval = Preconditions.checkNotNull(interval, "interval");
     this.groupingColumns = groupingColumns;
-    this.virtualColumns = virtualColumns;
+    this.virtualColumns = Preconditions.checkNotNull(virtualColumns, "virtualColumns");
     this.aggregators = aggregators;
-    this.orderByColumns = preferredOrdering;
-    this.queryContext = queryContext;
+    this.orderByColumns = Preconditions.checkNotNull(preferredOrdering, "preferredOrdering");
+    this.queryContext = Preconditions.checkNotNull(queryContext, "queryContext");
     this.queryMetrics = queryMetrics;
   }
 
@@ -139,7 +140,6 @@ public class CursorBuildSpec
    * <p>
    * If not specified, the cursor will advance in the native order of the underlying data.
    */
-  @Nullable
   public List<OrderBy> getPreferredOrdering()
   {
     return orderByColumns;
@@ -177,8 +177,7 @@ public class CursorBuildSpec
     private VirtualColumns virtualColumns = VirtualColumns.EMPTY;
     @Nullable
     private List<AggregatorFactory> aggregators;
-    @Nullable
-    private List<OrderBy> preferredOrdering;
+    private List<OrderBy> preferredOrdering = Collections.emptyList();
 
     private QueryContext queryContext = QueryContext.empty();
     @Nullable
@@ -251,9 +250,9 @@ public class CursorBuildSpec
     /**
      * @see CursorBuildSpec#getPreferredOrdering()
      */
-    public CursorBuildSpecBuilder setPreferredOrdering(@Nullable List<OrderBy> orderBy)
+    public CursorBuildSpecBuilder setPreferredOrdering(List<OrderBy> preferredOrdering)
     {
-      this.preferredOrdering = orderBy;
+      this.preferredOrdering = preferredOrdering;
       return this;
     }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/CursorHolder.java b/processing/src/main/java/org/apache/druid/segment/CursorHolder.java
index d0384e265b3..a70fd8757e1 100644
--- a/processing/src/main/java/org/apache/druid/segment/CursorHolder.java
+++ b/processing/src/main/java/org/apache/druid/segment/CursorHolder.java
@@ -20,7 +20,9 @@
 package org.apache.druid.segment;
 
 import org.apache.druid.java.util.common.UOE;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.OrderBy;
+import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.vector.VectorCursor;
 
 import javax.annotation.Nullable;
@@ -42,7 +44,10 @@ public interface CursorHolder extends Closeable
   @Nullable
   default VectorCursor asVectorCursor()
   {
-    throw new UOE("Cannot vectorize. Check 'canVectorize' before calling 'makeVectorCursor' on %s.", this.getClass().getName());
+    throw new UOE(
+        "Cannot vectorize. Check 'canVectorize' before calling 'makeVectorCursor' on %s.",
+        this.getClass().getName()
+    );
   }
 
   /**
@@ -55,13 +60,30 @@ public interface CursorHolder extends Closeable
 
   /**
    * Returns cursor ordering, which may or may not match {@link CursorBuildSpec#getPreferredOrdering()}. If returns
-   * null then the cursor has no defined ordering
+   * an empty list then the cursor has no defined ordering.
+   *
+   * Cursors associated with this holder return rows in this ordering, using the natural comparator for the column type.
+   * Includes {@link ColumnHolder#TIME_COLUMN_NAME} if appropriate.
    */
   default List<OrderBy> getOrdering()
   {
     return Collections.emptyList();
   }
 
+  /**
+   * If {@link #getOrdering()} starts with {@link ColumnHolder#TIME_COLUMN_NAME}, returns the time ordering; otherwise
+   * returns {@link Order#NONE}.
+   */
+  default Order getTimeOrder()
+  {
+    final List<OrderBy> ordering = getOrdering();
+    if (!ordering.isEmpty() && ColumnHolder.TIME_COLUMN_NAME.equals(ordering.get(0).getColumnName())) {
+      return ordering.get(0).getOrder();
+    } else {
+      return Order.NONE;
+    }
+  }
+
   /**
    * Release any resources acquired by cursors.
    */
diff --git a/processing/src/main/java/org/apache/druid/segment/Cursors.java b/processing/src/main/java/org/apache/druid/segment/Cursors.java
index 814681ab8b4..36b37ce3146 100644
--- a/processing/src/main/java/org/apache/druid/segment/Cursors.java
+++ b/processing/src/main/java/org/apache/druid/segment/Cursors.java
@@ -19,8 +19,6 @@
 
 package org.apache.druid.segment;
 
-import org.apache.druid.error.DruidException;
-import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.query.Order;
 import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.column.ColumnHolder;
@@ -54,34 +52,15 @@ public class Cursors
   }
 
   /**
-   * Check if the first {@link OrderBy} of {@link CursorHolder#getOrdering()} is {@link ColumnHolder#TIME_COLUMN_NAME}
-   * in the specified {@link Order}
+   * Return the {@link Order} of the {@link ColumnHolder#TIME_COLUMN_NAME}, based on a
+   * {@link CursorHolder#getOrdering()} or {@link Metadata#getOrdering()}.
    */
-  public static boolean isTimeOrdered(CursorHolder holder, Order order)
+  public static Order getTimeOrdering(final List<OrderBy> ordering)
   {
-    final List<OrderBy> ordering = holder.getOrdering();
-    if (ordering.isEmpty()) {
-      return false;
-    }
-    final OrderBy orderBy = ordering.get(0);
-    return ColumnHolder.TIME_COLUMN_NAME.equals(orderBy.getColumnName()) && orderBy.getOrder() == order;
-  }
-
-  /**
-   * Require the first {@link OrderBy} of {@link CursorHolder#getOrdering()} is {@link ColumnHolder#TIME_COLUMN_NAME}.
-   * Throws {@link DruidException} if the order does
-   */
-  public static void requireTimeOrdering(CursorHolder holder, Order order)
-  {
-    if (!isTimeOrdered(holder, order)) {
-      final String failureReason = StringUtils.format(
-          "Cursor must be ordered by [%s] with direction [%s] but was [%s] instead.",
-          ColumnHolder.TIME_COLUMN_NAME,
-          order,
-          holder.getOrdering()
-      );
-      holder.close();
-      throw DruidException.defensive(failureReason);
+    if (!ordering.isEmpty() && ColumnHolder.TIME_COLUMN_NAME.equals(ordering.get(0).getColumnName())) {
+      return ordering.get(0).getOrder();
+    } else {
+      return Order.NONE;
     }
   }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
index b4b0ddfbb60..3fcfe2dd78a 100644
--- a/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
@@ -25,7 +25,6 @@ import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.filter.AndFilter;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -88,18 +87,6 @@ public class FilteredStorageAdapter implements StorageAdapter
     return baseStorageAdapter.getDimensionCardinality(column);
   }
 
-  @Override
-  public DateTime getMinTime()
-  {
-    return baseStorageAdapter.getMinTime();
-  }
-
-  @Override
-  public DateTime getMaxTime()
-  {
-    return baseStorageAdapter.getMaxTime();
-  }
-
   @Nullable
   @Override
   public Comparable getMinValue(String column)
@@ -127,12 +114,6 @@ public class FilteredStorageAdapter implements StorageAdapter
     return 0;
   }
 
-  @Override
-  public DateTime getMaxIngestedEventTime()
-  {
-    return baseStorageAdapter.getMaxIngestedEventTime();
-  }
-
   @Nullable
   @Override
   public Metadata getMetadata()
diff --git a/processing/src/main/java/org/apache/druid/segment/IncrementalIndexMaxIngestedEventTimeInspector.java b/processing/src/main/java/org/apache/druid/segment/IncrementalIndexMaxIngestedEventTimeInspector.java
new file mode 100644
index 00000000000..5d4d10a37e6
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/IncrementalIndexMaxIngestedEventTimeInspector.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import org.apache.druid.segment.incremental.IncrementalIndex;
+import org.joda.time.DateTime;
+
+public class IncrementalIndexMaxIngestedEventTimeInspector implements MaxIngestedEventTimeInspector
+{
+  private final IncrementalIndex index;
+
+  public IncrementalIndexMaxIngestedEventTimeInspector(IncrementalIndex index)
+  {
+    this.index = index;
+  }
+
+  @Override
+  public DateTime getMaxIngestedEventTime()
+  {
+    return index.getMaxIngestedEventTime();
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/IncrementalIndexSegment.java b/processing/src/main/java/org/apache/druid/segment/IncrementalIndexSegment.java
index b270b54c4d1..62ecda675ac 100644
--- a/processing/src/main/java/org/apache/druid/segment/IncrementalIndexSegment.java
+++ b/processing/src/main/java/org/apache/druid/segment/IncrementalIndexSegment.java
@@ -24,6 +24,8 @@ import org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter;
 import org.apache.druid.timeline.SegmentId;
 import org.joda.time.Interval;
 
+import javax.annotation.Nullable;
+
 /**
  */
 public class IncrementalIndexSegment implements Segment
@@ -61,6 +63,19 @@ public class IncrementalIndexSegment implements Segment
     return new IncrementalIndexStorageAdapter(index);
   }
 
+  @Nullable
+  @Override
+  public <T> T as(final Class<T> clazz)
+  {
+    if (TimeBoundaryInspector.class.equals(clazz)) {
+      return (T) new IncrementalIndexTimeBoundaryInspector(index);
+    } else if (MaxIngestedEventTimeInspector.class.equals(clazz)) {
+      return (T) new IncrementalIndexMaxIngestedEventTimeInspector(index);
+    } else {
+      return Segment.super.as(clazz);
+    }
+  }
+
   @Override
   public void close()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/IncrementalIndexTimeBoundaryInspector.java b/processing/src/main/java/org/apache/druid/segment/IncrementalIndexTimeBoundaryInspector.java
new file mode 100644
index 00000000000..cfe8aa7d762
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/IncrementalIndexTimeBoundaryInspector.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.segment.incremental.IncrementalIndex;
+import org.joda.time.DateTime;
+
+/**
+ * {@link TimeBoundaryInspector} for {@link IncrementalIndex}.
+ */
+public class IncrementalIndexTimeBoundaryInspector implements TimeBoundaryInspector
+{
+  private final IncrementalIndex index;
+
+  public IncrementalIndexTimeBoundaryInspector(IncrementalIndex index)
+  {
+    this.index = index;
+  }
+
+  @Override
+  public DateTime getMinTime()
+  {
+    final DateTime minTime = index.getMinTime();
+    return minTime == null ? DateTimes.MIN : minTime;
+  }
+
+  @Override
+  public DateTime getMaxTime()
+  {
+    final DateTime maxTime = index.getMaxTime();
+    return maxTime == null ? DateTimes.MAX : maxTime;
+  }
+
+  @Override
+  public boolean isMinMaxExact()
+  {
+    return !index.isEmpty();
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java
index 966de405206..4db49535ae6 100644
--- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java
+++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java
@@ -140,8 +140,8 @@ public class IndexIO
       );
     }
     {
-      final Set<String> dimNames1 = Sets.newHashSet(adapter1.getDimensionNames());
-      final Set<String> dimNames2 = Sets.newHashSet(adapter2.getDimensionNames());
+      final Set<String> dimNames1 = Sets.newHashSet(adapter1.getDimensionNames(true));
+      final Set<String> dimNames2 = Sets.newHashSet(adapter2.getDimensionNames(true));
       if (!dimNames1.equals(dimNames2)) {
         throw new SegmentValidationException(
             "Dimension names differ. Expected [%s] found [%s]",
@@ -240,8 +240,8 @@ public class IndexIO
     if (dims1.size() != dims2.size()) {
       throw new SegmentValidationException("Dim lengths not equal %s vs %s", dims1, dims2);
     }
-    final List<String> dim1Names = adapter1.getDimensionNames();
-    final List<String> dim2Names = adapter2.getDimensionNames();
+    final List<String> dim1Names = adapter1.getDimensionNames(false);
+    final List<String> dim2Names = adapter2.getDimensionNames(false);
     int dimCount = dims1.size();
     for (int i = 0; i < dimCount; ++i) {
       final String dim1Name = dim1Names.get(i);
diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMerger.java b/processing/src/main/java/org/apache/druid/segment/IndexMerger.java
index c3e0e2d4dd9..f397b02814c 100644
--- a/processing/src/main/java/org/apache/druid/segment/IndexMerger.java
+++ b/processing/src/main/java/org/apache/druid/segment/IndexMerger.java
@@ -20,7 +20,6 @@
 package org.apache.druid.segment;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.inject.ImplementedBy;
@@ -29,6 +28,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.java.util.common.guava.Comparators;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.incremental.IncrementalIndex;
 import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory;
 import org.apache.druid.utils.CollectionUtils;
@@ -53,12 +53,18 @@ public interface IndexMerger
   int INVALID_ROW = -1;
   int UNLIMITED_MAX_COLUMNS_TO_MERGE = -1;
 
+  /**
+   * Returns merged dimensions, not including {@link ColumnHolder#TIME_COLUMN_NAME}, from a set of indexes.
+   */
   static List<String> getMergedDimensionsFromQueryableIndexes(
       List<QueryableIndex> indexes,
       @Nullable DimensionsSpec dimensionsSpec
   )
   {
-    return getMergedDimensions(toIndexableAdapters(indexes), dimensionsSpec);
+    return getMergedDimensionsWithTime(toIndexableAdapters(indexes), dimensionsSpec)
+        .stream()
+        .filter(column -> !ColumnHolder.TIME_COLUMN_NAME.equals(column))
+        .collect(Collectors.toList());
   }
 
   static List<IndexableAdapter> toIndexableAdapters(List<QueryableIndex> indexes)
@@ -66,12 +72,16 @@ public interface IndexMerger
     return indexes.stream().map(QueryableIndexIndexableAdapter::new).collect(Collectors.toList());
   }
 
-  static List<String> getMergedDimensions(
+  /**
+   * Returns a merged list of dimensions, including {@link ColumnHolder#TIME_COLUMN_NAME}, that will be used
+   * to build {@link TimeAndDimsPointer}.
+   */
+  static List<String> getMergedDimensionsWithTime(
       List<IndexableAdapter> indexes,
       @Nullable DimensionsSpec dimensionsSpec
   )
   {
-    if (indexes.size() == 0) {
+    if (indexes.isEmpty()) {
       return ImmutableList.of();
     }
     List<String> commonDimOrder = getLongestSharedDimOrder(indexes, dimensionsSpec);
@@ -84,6 +94,9 @@ public interface IndexMerger
     }
   }
 
+  /**
+   * Gets the longest shared dimension order, including both {@link ColumnHolder#TIME_COLUMN_NAME} and dimensions.
+   */
   @Nullable
   static List<String> getLongestSharedDimOrder(
       List<IndexableAdapter> indexes,
@@ -93,10 +106,10 @@ public interface IndexMerger
     int maxSize = 0;
     Iterable<String> orderingCandidate = null;
     for (IndexableAdapter index : indexes) {
-      int iterSize = index.getDimensionNames().size();
+      int iterSize = index.getDimensionNames(true).size();
       if (iterSize > maxSize) {
         maxSize = iterSize;
-        orderingCandidate = index.getDimensionNames();
+        orderingCandidate = index.getDimensionNames(true);
       }
     }
 
@@ -113,11 +126,16 @@ public interface IndexMerger
         log.info("Cannot fall back on dimension ordering from ingestionSpec as it does not exist");
         return null;
       }
-      List<String> candidate = new ArrayList<>(dimensionsSpec.getDimensionNames());
+      List<String> candidate = new ArrayList<>();
+      if (!dimensionsSpec.getDimensionNames().contains(ColumnHolder.TIME_COLUMN_NAME)) {
+        candidate.add(ColumnHolder.TIME_COLUMN_NAME);
+      }
+      candidate.addAll(dimensionsSpec.getDimensionNames());
       // Remove all dimensions that does not exist within the indexes from the candidate
-      Set<String> allValidDimensions = indexes.stream()
-                                         .flatMap(indexableAdapter -> indexableAdapter.getDimensionNames().stream())
-                                         .collect(Collectors.toSet());
+      Set<String> allValidDimensions =
+          indexes.stream()
+                 .flatMap(indexableAdapter -> indexableAdapter.getDimensionNames(true).stream())
+                 .collect(Collectors.toSet());
       candidate.retainAll(allValidDimensions);
       // Sanity check that there is no extra/missing columns
       if (candidate.size() != allValidDimensions.size()) {
@@ -141,7 +159,7 @@ public interface IndexMerger
   {
     for (IndexableAdapter index : indexes) {
       Iterator<String> candidateIter = orderingCandidate.iterator();
-      for (String matchDim : index.getDimensionNames()) {
+      for (String matchDim : index.getDimensionNames(true)) {
         boolean matched = false;
         while (candidateIter.hasNext()) {
           String nextDim = candidateIter.next();
@@ -158,21 +176,15 @@ public interface IndexMerger
     return true;
   }
 
+  /**
+   * Returns {@link ColumnHolder#TIME_COLUMN_NAME} followed by a merged list of dimensions in lexicographic order.
+   */
   static List<String> getLexicographicMergedDimensions(List<IndexableAdapter> indexes)
   {
-    return mergeIndexed(
-        Lists.transform(
-            indexes,
-            new Function<IndexableAdapter, Iterable<String>>()
-            {
-              @Override
-              public Iterable<String> apply(@Nullable IndexableAdapter input)
-              {
-                return input.getDimensionNames();
-              }
-            }
-        )
-    );
+    final List<String> retVal = new ArrayList<>();
+    retVal.add(ColumnHolder.TIME_COLUMN_NAME);
+    retVal.addAll(mergeIndexed(Lists.transform(indexes, input -> input.getDimensionNames(false))));
+    return retVal;
   }
 
   static <T extends Comparable<? super T>> ArrayList<T> mergeIndexed(List<Iterable<T>> indexedLists)
diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java
index 05c721e5c56..c60a4acf351 100644
--- a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java
+++ b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java
@@ -28,6 +28,7 @@ import com.google.common.primitives.Ints;
 import com.google.inject.Inject;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.io.ZeroCopyByteArrayOutputStream;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.FileUtils;
@@ -133,7 +134,7 @@ public class IndexMergerV9 implements IndexMerger
       final @Nullable AggregatorFactory[] metricAggs,
       final File outDir,
       final ProgressIndicator progress,
-      final List<String> mergedDimensions, // should have both explicit and implicit dimensions
+      final List<String> mergedDimensionsWithTime, // has both explicit and implicit dimensions, as well as __time
       final DimensionsSpecInspector dimensionsSpecInspector,
       final List<String> mergedMetrics,
       final Function<List<TransformableRowIterator>, TimeAndDimsIterator> rowMergerFn,
@@ -147,6 +148,12 @@ public class IndexMergerV9 implements IndexMerger
 
     List<Metadata> metadataList = Lists.transform(adapters, IndexableAdapter::getMetadata);
 
+    // Merged dimensions without __time.
+    List<String> mergedDimensions =
+        mergedDimensionsWithTime.stream()
+                                .filter(dim -> !ColumnHolder.TIME_COLUMN_NAME.equals(dim))
+                                .collect(Collectors.toList());
+
     final Metadata segmentMetadata;
     if (metricAggs != null) {
       AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[metricAggs.length];
@@ -164,6 +171,18 @@ public class IndexMergerV9 implements IndexMerger
       );
     }
 
+    if (segmentMetadata != null
+        && segmentMetadata.getOrdering() != null
+        && segmentMetadata.getOrdering()
+                          .stream()
+                          .noneMatch(orderBy -> ColumnHolder.TIME_COLUMN_NAME.equals(orderBy.getColumnName()))) {
+      throw DruidException.defensive(
+          "sortOrder[%s] must include[%s]",
+          segmentMetadata.getOrdering(),
+          ColumnHolder.TIME_COLUMN_NAME
+      );
+    }
+
     Closer closer = Closer.create();
     try {
       final FileSmoosher v9Smoosher = new FileSmoosher(outDir);
@@ -220,7 +239,7 @@ public class IndexMergerV9 implements IndexMerger
       progress.progress();
       final TimeAndDimsIterator timeAndDimsIterator = makeMergedTimeAndDimsIterator(
           adapters,
-          mergedDimensions,
+          mergedDimensionsWithTime,
           mergedMetrics,
           rowMergerFn,
           handlers,
@@ -827,7 +846,7 @@ public class IndexMergerV9 implements IndexMerger
   {
     final Map<String, ColumnFormat> columnFormats = new HashMap<>();
     for (IndexableAdapter adapter : adapters) {
-      for (String dimension : adapter.getDimensionNames()) {
+      for (String dimension : adapter.getDimensionNames(false)) {
         ColumnFormat format = adapter.getFormat(dimension);
         columnFormats.compute(dimension, (d, existingFormat) -> existingFormat == null ? format : format.merge(existingFormat));
       }
@@ -1082,8 +1101,7 @@ public class IndexMergerV9 implements IndexMerger
 
   private int getIndexColumnCount(IndexableAdapter indexableAdapter)
   {
-    // +1 for the __time column
-    return 1 + indexableAdapter.getDimensionNames().size() + indexableAdapter.getMetricNames().size();
+    return indexableAdapter.getDimensionNames(true).size() + indexableAdapter.getMetricNames().size();
   }
 
   private int getIndexColumnCount(List<IndexableAdapter> indexableAdapters)
@@ -1106,7 +1124,7 @@ public class IndexMergerV9 implements IndexMerger
       @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory
   ) throws IOException
   {
-    final List<String> mergedDimensions = IndexMerger.getMergedDimensions(indexes, dimensionsSpec);
+    final List<String> mergedDimensionsWithTime = IndexMerger.getMergedDimensionsWithTime(indexes, dimensionsSpec);
 
     final List<String> mergedMetrics = IndexMerger.mergeIndexed(
         indexes.stream().map(IndexableAdapter::getMetricNames).collect(Collectors.toList())
@@ -1157,7 +1175,7 @@ public class IndexMergerV9 implements IndexMerger
         sortedMetricAggs,
         outDir,
         progress,
-        mergedDimensions,
+        mergedDimensionsWithTime,
         new DimensionsSpecInspector(storeEmptyColumns, dimensionsSpec),
         mergedMetrics,
         rowMergerFn,
@@ -1183,7 +1201,7 @@ public class IndexMergerV9 implements IndexMerger
 
   private TimeAndDimsIterator makeMergedTimeAndDimsIterator(
       final List<IndexableAdapter> indexes,
-      final List<String> mergedDimensions,
+      final List<String> mergedDimensionsWithTime,
       final List<String> mergedMetrics,
       final Function<List<TransformableRowIterator>, TimeAndDimsIterator> rowMergerFn,
       final Map<String, DimensionHandler> handlers,
@@ -1194,9 +1212,10 @@ public class IndexMergerV9 implements IndexMerger
     for (int i = 0; i < indexes.size(); ++i) {
       final IndexableAdapter adapter = indexes.get(i);
       TransformableRowIterator target = adapter.getRows();
-      if (!mergedDimensions.equals(adapter.getDimensionNames()) || !mergedMetrics.equals(adapter.getMetricNames())) {
+      if (!mergedDimensionsWithTime.equals(adapter.getDimensionNames(true))
+          || !mergedMetrics.equals(adapter.getMetricNames())) {
         target = makeRowIteratorWithReorderedColumns(
-            mergedDimensions,
+            mergedDimensionsWithTime,
             mergedMetrics,
             handlers,
             adapter,
@@ -1209,7 +1228,7 @@ public class IndexMergerV9 implements IndexMerger
   }
 
   private TransformableRowIterator makeRowIteratorWithReorderedColumns(
-      List<String> reorderedDimensions,
+      List<String> reorderedDimensionsWithTime,
       List<String> reorderedMetrics,
       Map<String, DimensionHandler> originalHandlers,
       IndexableAdapter originalAdapter,
@@ -1217,14 +1236,14 @@ public class IndexMergerV9 implements IndexMerger
   )
   {
     RowPointer reorderedRowPointer = reorderRowPointerColumns(
-        reorderedDimensions,
+        reorderedDimensionsWithTime,
         reorderedMetrics,
         originalHandlers,
         originalAdapter,
         originalIterator.getPointer()
     );
     TimeAndDimsPointer reorderedMarkedRowPointer = reorderRowPointerColumns(
-        reorderedDimensions,
+        reorderedDimensionsWithTime,
         reorderedMetrics,
         originalHandlers,
         originalAdapter,
@@ -1247,17 +1266,22 @@ public class IndexMergerV9 implements IndexMerger
   }
 
   private static <T extends TimeAndDimsPointer> T reorderRowPointerColumns(
-      List<String> reorderedDimensions,
+      List<String> reorderedDimensionsWithTime,
       List<String> reorderedMetrics,
       Map<String, DimensionHandler> originalHandlers,
       IndexableAdapter originalAdapter,
       T originalRowPointer
   )
   {
-    ColumnValueSelector[] reorderedDimensionSelectors = reorderedDimensions
+    int reorderedTimePosition = reorderedDimensionsWithTime.indexOf(ColumnHolder.TIME_COLUMN_NAME);
+    if (reorderedTimePosition < 0) {
+      throw DruidException.defensive("Missing column[%s]", ColumnHolder.TIME_COLUMN_NAME);
+    }
+    ColumnValueSelector[] reorderedDimensionSelectors = reorderedDimensionsWithTime
         .stream()
+        .filter(column -> !ColumnHolder.TIME_COLUMN_NAME.equals(column))
         .map(dimName -> {
-          int dimIndex = originalAdapter.getDimensionNames().indexOf(dimName);
+          int dimIndex = originalAdapter.getDimensionNames(false).indexOf(dimName);
           if (dimIndex >= 0) {
             return originalRowPointer.getDimensionSelector(dimIndex);
           } else {
@@ -1266,7 +1290,9 @@ public class IndexMergerV9 implements IndexMerger
         })
         .toArray(ColumnValueSelector[]::new);
     List<DimensionHandler> reorderedHandlers =
-        reorderedDimensions.stream().map(originalHandlers::get).collect(Collectors.toList());
+        reorderedDimensionsWithTime.stream()
+                                   .filter(column -> !ColumnHolder.TIME_COLUMN_NAME.equals(column))
+                                   .map(originalHandlers::get).collect(Collectors.toList());
     ColumnValueSelector[] reorderedMetricSelectors = reorderedMetrics
         .stream()
         .map(metricName -> {
@@ -1282,6 +1308,7 @@ public class IndexMergerV9 implements IndexMerger
       //noinspection unchecked
       return (T) new RowPointer(
           originalRowPointer.timestampSelector,
+          reorderedTimePosition,
           reorderedDimensionSelectors,
           reorderedHandlers,
           reorderedMetricSelectors,
@@ -1292,6 +1319,7 @@ public class IndexMergerV9 implements IndexMerger
       //noinspection unchecked
       return (T) new TimeAndDimsPointer(
           originalRowPointer.timestampSelector,
+          reorderedTimePosition,
           reorderedDimensionSelectors,
           reorderedHandlers,
           reorderedMetricSelectors,
diff --git a/processing/src/main/java/org/apache/druid/segment/IndexableAdapter.java b/processing/src/main/java/org/apache/druid/segment/IndexableAdapter.java
index 6f520dd2f9c..023ca700192 100644
--- a/processing/src/main/java/org/apache/druid/segment/IndexableAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/IndexableAdapter.java
@@ -23,6 +23,7 @@ import com.google.errorprone.annotations.MustBeClosed;
 import org.apache.druid.segment.column.CapabilitiesBasedFormat;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnFormat;
+import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.data.BitmapValues;
 import org.apache.druid.segment.data.CloseableIndexed;
 import org.apache.druid.segment.nested.FieldTypeInfo;
@@ -45,7 +46,12 @@ public interface IndexableAdapter
 
   int getNumRows();
 
-  List<String> getDimensionNames();
+  /**
+   * Returns names of dimension columns.
+   *
+   * @param includeTime whether to include {@link ColumnHolder#TIME_COLUMN_NAME}.
+   */
+  List<String> getDimensionNames(boolean includeTime);
 
   List<String> getMetricNames();
 
diff --git a/processing/src/main/java/org/apache/druid/segment/MaxIngestedEventTimeInspector.java b/processing/src/main/java/org/apache/druid/segment/MaxIngestedEventTimeInspector.java
new file mode 100644
index 00000000000..67fc7536f11
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/MaxIngestedEventTimeInspector.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import org.apache.druid.data.input.InputRow;
+import org.apache.druid.query.datasourcemetadata.DataSourceMetadataQuery;
+import org.apache.druid.query.datasourcemetadata.DataSourceMetadataResultValue;
+import org.joda.time.DateTime;
+
+/**
+ * Inspector used by {@link DataSourceMetadataQuery} to populate
+ * {@link DataSourceMetadataResultValue#getMaxIngestedEventTime()}.
+ */
+public interface MaxIngestedEventTimeInspector
+{
+  /**
+   * Highest known {@link InputRow#getTimestamp()} for rows that went into creating this index.
+   */
+  DateTime getMaxIngestedEventTime();
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/Metadata.java b/processing/src/main/java/org/apache/druid/segment/Metadata.java
index c3ac554a0fa..e21c673e279 100644
--- a/processing/src/main/java/org/apache/druid/segment/Metadata.java
+++ b/processing/src/main/java/org/apache/druid/segment/Metadata.java
@@ -19,10 +19,13 @@
 
 package org.apache.druid.segment;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.guice.annotations.PublicApi;
+import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 
 import javax.annotation.Nullable;
@@ -35,6 +38,7 @@ import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
+ *
  */
 @PublicApi
 public class Metadata
@@ -50,13 +54,16 @@ public class Metadata
   private final Granularity queryGranularity;
   @Nullable
   private final Boolean rollup;
+  @Nullable
+  private final List<OrderBy> ordering;
 
   public Metadata(
       @JsonProperty("container") @Nullable Map<String, Object> container,
       @JsonProperty("aggregators") @Nullable AggregatorFactory[] aggregators,
       @JsonProperty("timestampSpec") @Nullable TimestampSpec timestampSpec,
       @JsonProperty("queryGranularity") @Nullable Granularity queryGranularity,
-      @JsonProperty("rollup") @Nullable Boolean rollup
+      @JsonProperty("rollup") @Nullable Boolean rollup,
+      @JsonProperty("ordering") @Nullable List<OrderBy> ordering
   )
   {
     this.container = container == null ? new ConcurrentHashMap<>() : container;
@@ -64,6 +71,7 @@ public class Metadata
     this.timestampSpec = timestampSpec;
     this.queryGranularity = queryGranularity;
     this.rollup = rollup;
+    this.ordering = ordering;
   }
 
   @JsonProperty
@@ -100,6 +108,22 @@ public class Metadata
     return rollup;
   }
 
+  /**
+   * Ordering for the segment associated with this object. Nonnull for segments written in current versions of Druid,
+   * but would null for older segments. Null may be interpreted as {@link Cursors#ascendingTimeOrder()}, since prior
+   * to this field being added, segments were always time-ordered.
+   *
+   * When dealing with {@link QueryableIndex}, it is generally better to use {@link QueryableIndex#getOrdering()}, since
+   * that method never returns null.
+   */
+  @Nullable
+  @JsonProperty
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  public List<OrderBy> getOrdering()
+  {
+    return ordering;
+  }
+
   public Metadata putAll(@Nullable Map<String, Object> other)
   {
     if (other != null) {
@@ -130,6 +154,7 @@ public class Metadata
     List<TimestampSpec> timestampSpecsToMerge = new ArrayList<>();
     List<Granularity> gransToMerge = new ArrayList<>();
     List<Boolean> rollupToMerge = new ArrayList<>();
+    List<List<OrderBy>> orderingsToMerge = new ArrayList<>();
 
     for (Metadata metadata : toBeMerged) {
       if (metadata != null) {
@@ -149,6 +174,8 @@ public class Metadata
         if (rollupToMerge != null) {
           rollupToMerge.add(metadata.isRollup());
         }
+
+        orderingsToMerge.add(metadata.getOrdering());
         mergedContainer.putAll(metadata.container);
       } else {
         //if metadata and hence aggregators and queryGranularity for some segment being merged are unknown then
@@ -176,6 +203,8 @@ public class Metadata
                                           null :
                                           Granularity.mergeGranularities(gransToMerge);
 
+    final List<OrderBy> mergedOrdering = mergeOrderings(orderingsToMerge);
+
     Boolean rollup = null;
     if (rollupToMerge != null && !rollupToMerge.isEmpty()) {
       rollup = rollupToMerge.get(0);
@@ -197,7 +226,8 @@ public class Metadata
         mergedAggregators,
         mergedTimestampSpec,
         mergedGranularity,
-        rollup
+        rollup,
+        mergedOrdering
     );
   }
 
@@ -215,7 +245,8 @@ public class Metadata
            Arrays.equals(aggregators, metadata.aggregators) &&
            Objects.equals(timestampSpec, metadata.timestampSpec) &&
            Objects.equals(queryGranularity, metadata.queryGranularity) &&
-           Objects.equals(rollup, metadata.rollup);
+           Objects.equals(rollup, metadata.rollup) &&
+           Objects.equals(ordering, metadata.ordering);
   }
 
   @Override
@@ -233,6 +264,48 @@ public class Metadata
            ", timestampSpec=" + timestampSpec +
            ", queryGranularity=" + queryGranularity +
            ", rollup=" + rollup +
+           ", ordering=" + ordering +
            '}';
   }
+
+  /**
+   * Merge {@link #getOrdering()} from different metadatas.
+   *
+   * When an input sort order is null, we assume it is {@link Cursors#ascendingTimeOrder()}, as this was the only
+   * sort order possible prior to the introduction of the "ordering" field.
+   */
+  public static List<OrderBy> mergeOrderings(List<List<OrderBy>> orderingsToMerge)
+  {
+    if (orderingsToMerge.isEmpty()) {
+      throw new IAE("orderingsToMerge is empty");
+    }
+
+    final List<OrderBy> mergedOrdering = new ArrayList<>();
+
+    while (true) {
+      final int position = mergedOrdering.size();
+      OrderBy orderBy = null;
+
+      // Iterate through each sort order, check that the columns at "position" are all the same. If not, return
+      // the mergedOrdering as-is.
+      for (List<OrderBy> ordering : orderingsToMerge) {
+        if (ordering == null) {
+          // null ordering is treated as [__time].
+          ordering = Cursors.ascendingTimeOrder();
+        }
+
+        if (position < ordering.size()) {
+          if (orderBy == null) {
+            orderBy = ordering.get(position);
+          } else if (!orderBy.equals(ordering.get(position))) {
+            return mergedOrdering;
+          }
+        } else {
+          return mergedOrdering;
+        }
+      }
+
+      mergedOrdering.add(orderBy);
+    }
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java
index b4cd503681c..3766db4a4b1 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java
@@ -20,6 +20,7 @@
 package org.apache.druid.segment;
 
 import org.apache.druid.collections.bitmap.BitmapFactory;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.data.Indexed;
@@ -42,9 +43,17 @@ public interface QueryableIndex extends Closeable, ColumnInspector
 {
   Interval getDataInterval();
   int getNumRows();
+  /**
+   * List of dimensions, not including {@link ColumnHolder#TIME_COLUMN_NAME}.
+   */
   Indexed<String> getAvailableDimensions();
   BitmapFactory getBitmapFactoryForDimensions();
   @Nullable Metadata getMetadata();
+
+  /**
+   * Map of column name to {@link DimensionHandler}, whose contents and iteration order matches
+   * {@link #getAvailableDimensions()}.
+   */
   Map<String, DimensionHandler> getDimensionHandlers();
 
   List<String> getColumnNames();
@@ -63,6 +72,11 @@ public interface QueryableIndex extends Closeable, ColumnInspector
     return columnHolder.getCapabilities();
   }
 
+  /**
+   * Returns the ordering of rows in this index.
+   */
+  List<OrderBy> getOrdering();
+
   /**
    * The close method shouldn't actually be here as this is nasty. We will adjust it in the future.
    * @throws IOException if an exception was thrown closing the index
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java
index 887f74368aa..22da9a07505 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.segment;
 
+import org.apache.druid.query.Order;
 import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.extraction.ExtractionFn;
 import org.apache.druid.segment.column.BaseColumn;
@@ -43,7 +44,7 @@ import java.util.function.Function;
 public class QueryableIndexColumnSelectorFactory implements ColumnSelectorFactory, RowIdSupplier
 {
   private final VirtualColumns virtualColumns;
-  private final boolean descending;
+  private final Order timeOrder;
   protected final ReadableOffset offset;
 
   // Share Column objects, since they cache decompressed buffers internally, and we can avoid recomputation if the
@@ -56,13 +57,13 @@ public class QueryableIndexColumnSelectorFactory implements ColumnSelectorFactor
 
   public QueryableIndexColumnSelectorFactory(
       VirtualColumns virtualColumns,
-      boolean descending,
+      Order timeOrder,
       ReadableOffset offset,
       ColumnCache columnCache
   )
   {
     this.virtualColumns = virtualColumns;
-    this.descending = descending;
+    this.timeOrder = timeOrder;
     this.offset = offset;
     this.columnCache = columnCache;
     this.dimensionSelectorCache = new HashMap<>();
@@ -107,8 +108,12 @@ public class QueryableIndexColumnSelectorFactory implements ColumnSelectorFactor
       return DimensionSelector.constant(null, extractionFn);
     }
 
-    if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME)) {
-      return new SingleScanTimeDimensionSelector(makeColumnValueSelector(dimension), extractionFn, descending);
+    if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME) && timeOrder != Order.NONE) {
+      return new SingleScanTimeDimensionSelector(
+          makeColumnValueSelector(dimension),
+          extractionFn,
+          timeOrder
+      );
     }
 
     ColumnCapabilities capabilities = columnHolder.getCapabilities();
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
index 0f0646839ac..53412bf9da0 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
@@ -22,6 +22,7 @@ package org.apache.druid.segment;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableList;
 import org.apache.druid.collections.bitmap.BitmapFactory;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.io.Closer;
@@ -29,6 +30,7 @@ import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.BitmapResultFactory;
 import org.apache.druid.query.DefaultBitmapResultFactory;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryContext;
@@ -36,13 +38,16 @@ import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.FilterBundle;
+import org.apache.druid.query.filter.RangeFilter;
 import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.query.filter.vector.VectorValueMatcher;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.NumericColumn;
 import org.apache.druid.segment.data.Offset;
 import org.apache.druid.segment.data.ReadableOffset;
+import org.apache.druid.segment.filter.AndFilter;
 import org.apache.druid.segment.historical.HistoricalCursor;
 import org.apache.druid.segment.vector.BitmapVectorOffset;
 import org.apache.druid.segment.vector.FilteredVectorOffset;
@@ -57,7 +62,6 @@ import org.joda.time.Interval;
 import javax.annotation.Nullable;
 import java.io.Closeable;
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
@@ -75,7 +79,6 @@ public class QueryableIndexCursorHolder implements CursorHolder
   @Nullable
   private final QueryMetrics<? extends Query> metrics;
   private final List<OrderBy> ordering;
-  private final boolean descending;
   private final QueryContext queryContext;
   private final int vectorSize;
   private final Supplier<CursorResources> resourcesSupplier;
@@ -90,18 +93,28 @@ public class QueryableIndexCursorHolder implements CursorHolder
     this.virtualColumns = cursorBuildSpec.getVirtualColumns();
     this.aggregatorFactories = cursorBuildSpec.getAggregators();
     this.filter = cursorBuildSpec.getFilter();
-    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
-    if (Cursors.preferDescendingTimeOrdering(cursorBuildSpec)) {
-      this.ordering = Collections.singletonList(OrderBy.descending(ColumnHolder.TIME_COLUMN_NAME));
-      this.descending = true;
+
+    final List<OrderBy> indexOrdering = index.getOrdering();
+    if (Cursors.preferDescendingTimeOrdering(cursorBuildSpec)
+        && Cursors.getTimeOrdering(indexOrdering) == Order.ASCENDING) {
+      this.ordering = Cursors.descendingTimeOrder();
     } else {
-      this.ordering = Collections.singletonList(OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME));
-      this.descending = false;
+      this.ordering = indexOrdering;
     }
+
     this.queryContext = cursorBuildSpec.getQueryContext();
     this.vectorSize = cursorBuildSpec.getQueryContext().getVectorSize();
     this.metrics = cursorBuildSpec.getQueryMetrics();
-    this.resourcesSupplier = Suppliers.memoize(() -> new CursorResources(index, virtualColumns, filter, metrics));
+    this.resourcesSupplier = Suppliers.memoize(
+        () -> new CursorResources(
+            index,
+            virtualColumns,
+            Cursors.getTimeOrdering(ordering),
+            interval,
+            filter,
+            metrics
+        )
+    );
   }
 
   @Override
@@ -130,7 +143,7 @@ public class QueryableIndexCursorHolder implements CursorHolder
     }
 
     // vector cursors can't iterate backwards yet
-    return !descending;
+    return Cursors.getTimeOrdering(ordering) != Order.DESCENDING;
   }
 
   @Override
@@ -148,51 +161,60 @@ public class QueryableIndexCursorHolder implements CursorHolder
     final long maxDataTimestamp = resources.maxDataTimestamp;
     final NumericColumn timestamps = resources.timestamps;
     final ColumnCache columnCache = resources.columnCache;
+    final Order timeOrder = resources.timeOrder;
 
-    // filterBundle will only be null if the filter itself is null, otherwise check to see if the filter
+    // if filterBundle is null, the filter itself is also null. otherwise check to see if the filter
     // can use an index
     if (filterBundle == null || filterBundle.getIndex() == null) {
-      baseOffset = descending ? new SimpleDescendingOffset(numRows) : new SimpleAscendingOffset(numRows);
+      baseOffset =
+          timeOrder == Order.DESCENDING ? new SimpleDescendingOffset(numRows) : new SimpleAscendingOffset(numRows);
     } else {
-      baseOffset = BitmapOffset.of(filterBundle.getIndex().getBitmap(), descending, index.getNumRows());
+      baseOffset =
+          BitmapOffset.of(filterBundle.getIndex().getBitmap(), timeOrder == Order.DESCENDING, index.getNumRows());
     }
 
     final long timeStart = Math.max(interval.getStartMillis(), minDataTimestamp);
     final long timeEnd = interval.getEndMillis();
 
-    if (descending) {
-      for (; baseOffset.withinBounds(); baseOffset.increment()) {
-        if (timestamps.getLongSingleValueRow(baseOffset.getOffset()) < timeEnd) {
-          break;
-        }
-      }
-    } else {
+    if (timeOrder == Order.ASCENDING) {
       for (; baseOffset.withinBounds(); baseOffset.increment()) {
         if (timestamps.getLongSingleValueRow(baseOffset.getOffset()) >= timeStart) {
           break;
         }
       }
+    } else if (timeOrder == Order.DESCENDING) {
+      for (; baseOffset.withinBounds(); baseOffset.increment()) {
+        if (timestamps.getLongSingleValueRow(baseOffset.getOffset()) < timeEnd) {
+          break;
+        }
+      }
     }
 
-    final Offset offset = descending ?
-                          new DescendingTimestampCheckingOffset(
-                              baseOffset,
-                              timestamps,
-                              timeStart,
-                              minDataTimestamp >= timeStart
-                          ) :
-                          new AscendingTimestampCheckingOffset(
-                              baseOffset,
-                              timestamps,
-                              timeEnd,
-                              maxDataTimestamp < timeEnd
-                          );
+    final Offset offset;
 
+    if (timeOrder == Order.ASCENDING) {
+      offset = new AscendingTimestampCheckingOffset(
+          baseOffset,
+          timestamps,
+          timeEnd,
+          maxDataTimestamp < timeEnd
+      );
+    } else if (timeOrder == Order.DESCENDING) {
+      offset = new DescendingTimestampCheckingOffset(
+          baseOffset,
+          timestamps,
+          timeStart,
+          minDataTimestamp >= timeStart
+      );
+    } else {
+      // Time filter is moved into filterBundle in the non-time-ordered case.
+      offset = baseOffset;
+    }
 
     final Offset baseCursorOffset = offset.clone();
     final ColumnSelectorFactory columnSelectorFactory = new QueryableIndexColumnSelectorFactory(
         virtualColumns,
-        descending,
+        Cursors.getTimeOrdering(ordering),
         baseCursorOffset.getBaseReadableOffset(),
         columnCache
     );
@@ -203,7 +225,7 @@ public class QueryableIndexCursorHolder implements CursorHolder
                                                .valueMatcher(
                                                    columnSelectorFactory,
                                                    baseCursorOffset,
-                                                   descending
+                                                   timeOrder == Order.DESCENDING
                                                );
       final FilteredOffset filteredOffset = new FilteredOffset(baseCursorOffset, matcher);
       return new QueryableIndexCursor(filteredOffset, columnSelectorFactory);
@@ -212,6 +234,10 @@ public class QueryableIndexCursorHolder implements CursorHolder
     }
   }
 
+  /**
+   * Compute filter to use for cursor creation. For non-time-ordered segments, this includes the query interval
+   * as a filter.
+   */
   @Nullable
   @Override
   public VectorCursor asVectorCursor()
@@ -222,6 +248,7 @@ public class QueryableIndexCursorHolder implements CursorHolder
     final long maxDataTimestamp = resources.maxDataTimestamp;
     final NumericColumn timestamps = resources.timestamps;
     final ColumnCache columnCache = resources.columnCache;
+    final Order timeOrder = resources.timeOrder;
     // Wrap the remainder of cursor setup in a try, so if an error is encountered while setting it up, we don't
     // leak columns in the ColumnCache.
 
@@ -234,17 +261,18 @@ public class QueryableIndexCursorHolder implements CursorHolder
       metrics.vectorized(true);
     }
 
-
+    // startOffset, endOffset must match the "interval" if timeOrdered. Otherwise, the "interval" filtering is embedded
+    // within the filterBundle.
     final int startOffset;
     final int endOffset;
 
-    if (interval.getStartMillis() > minDataTimestamp) {
+    if (timeOrder != Order.NONE && interval.getStartMillis() > minDataTimestamp) {
       startOffset = timeSearch(timestamps, interval.getStartMillis(), 0, index.getNumRows());
     } else {
       startOffset = 0;
     }
 
-    if (interval.getEndMillis() <= maxDataTimestamp) {
+    if (timeOrder != Order.NONE && interval.getEndMillis() <= maxDataTimestamp) {
       endOffset = timeSearch(timestamps, interval.getEndMillis(), startOffset, index.getNumRows());
     } else {
       endOffset = index.getNumRows();
@@ -318,6 +346,7 @@ public class QueryableIndexCursorHolder implements CursorHolder
    * @param timestamp  the timestamp to search for
    * @param startIndex first index to search, inclusive
    * @param endIndex   last index to search, exclusive
+   *
    * @return first index that has a timestamp equal to, or greater, than "timestamp"
    */
   @VisibleForTesting
@@ -625,24 +654,21 @@ public class QueryableIndexCursorHolder implements CursorHolder
     private final long maxDataTimestamp;
     private final int numRows;
     @Nullable
-    private final Filter filter;
-    @Nullable
     private final FilterBundle filterBundle;
     private final NumericColumn timestamps;
+    private final Order timeOrder;
     private final ColumnCache columnCache;
-    @Nullable
-    private final QueryMetrics<? extends Query<?>> metrics;
 
     private CursorResources(
         QueryableIndex index,
         VirtualColumns virtualColumns,
+        Order timeOrder,
+        Interval interval,
         @Nullable Filter filter,
         @Nullable QueryMetrics<? extends Query<?>> metrics
     )
     {
       this.closer = Closer.create();
-      this.filter = filter;
-      this.metrics = metrics;
       this.columnCache = new ColumnCache(index, closer);
       final ColumnSelectorColumnIndexSelector bitmapIndexSelector = new ColumnSelectorColumnIndexSelector(
           index.getBitmapFactoryForDimensions(),
@@ -651,10 +677,22 @@ public class QueryableIndexCursorHolder implements CursorHolder
       );
       try {
         this.numRows = index.getNumRows();
-        this.filterBundle = makeFilterBundle(bitmapIndexSelector, numRows);
         this.timestamps = (NumericColumn) columnCache.getColumn(ColumnHolder.TIME_COLUMN_NAME);
         this.minDataTimestamp = DateTimes.utc(timestamps.getLongSingleValueRow(0)).getMillis();
         this.maxDataTimestamp = DateTimes.utc(timestamps.getLongSingleValueRow(timestamps.length() - 1)).getMillis();
+        this.filterBundle = makeFilterBundle(
+            computeFilterWithIntervalIfNeeded(
+                timeOrder,
+                this.minDataTimestamp,
+                this.maxDataTimestamp,
+                interval,
+                filter
+            ),
+            bitmapIndexSelector,
+            numRows,
+            metrics
+        );
+        this.timeOrder = timeOrder;
       }
       catch (Throwable t) {
         throw CloseableUtils.closeAndWrapInCatch(t, closer);
@@ -666,49 +704,93 @@ public class QueryableIndexCursorHolder implements CursorHolder
     {
       closer.close();
     }
+  }
 
-    @Nullable
-    private FilterBundle makeFilterBundle(
-        ColumnSelectorColumnIndexSelector bitmapIndexSelector,
-        int numRows
-    )
-    {
-      final BitmapFactory bitmapFactory = bitmapIndexSelector.getBitmapFactory();
-      final BitmapResultFactory<?> bitmapResultFactory;
-      if (metrics != null) {
-        bitmapResultFactory = metrics.makeBitmapResultFactory(bitmapFactory);
-        metrics.reportSegmentRows(numRows);
+  /**
+   * Create a {@link FilterBundle} for a cursor hold instance.
+   *
+   * The provided filter must include the query-level interface if needed. To compute this properly, use
+   * {@link #computeFilterWithIntervalIfNeeded}.
+   */
+  @Nullable
+  private static FilterBundle makeFilterBundle(
+      @Nullable final Filter filter,
+      final ColumnSelectorColumnIndexSelector bitmapIndexSelector,
+      final int numRows,
+      @Nullable final QueryMetrics<?> metrics
+  )
+  {
+    final BitmapFactory bitmapFactory = bitmapIndexSelector.getBitmapFactory();
+    final BitmapResultFactory<?> bitmapResultFactory;
+    if (metrics != null) {
+      bitmapResultFactory = metrics.makeBitmapResultFactory(bitmapFactory);
+      metrics.reportSegmentRows(numRows);
+    } else {
+      bitmapResultFactory = new DefaultBitmapResultFactory(bitmapFactory);
+    }
+    if (filter == null) {
+      return null;
+    }
+    final long bitmapConstructionStartNs = System.nanoTime();
+    final FilterBundle filterBundle = filter.makeFilterBundle(
+        bitmapIndexSelector,
+        bitmapResultFactory,
+        numRows,
+        numRows,
+        false
+    );
+    if (metrics != null) {
+      final long buildTime = System.nanoTime() - bitmapConstructionStartNs;
+      metrics.reportBitmapConstructionTime(buildTime);
+      final FilterBundle.BundleInfo info = filterBundle.getInfo();
+      metrics.filterBundle(info);
+      log.debug("Filter partitioning (%sms):%s", TimeUnit.NANOSECONDS.toMillis(buildTime), info);
+      if (filterBundle.getIndex() != null) {
+        metrics.reportPreFilteredRows(filterBundle.getIndex().getBitmap().size());
       } else {
-        bitmapResultFactory = new DefaultBitmapResultFactory(bitmapFactory);
+        metrics.reportPreFilteredRows(0);
       }
-      if (filter == null) {
-        return null;
-      }
-      final long bitmapConstructionStartNs = System.nanoTime();
-      final FilterBundle filterBundle = filter.makeFilterBundle(
-          bitmapIndexSelector,
-          bitmapResultFactory,
-          numRows,
-          numRows,
-          false
+    } else if (log.isDebugEnabled()) {
+      final FilterBundle.BundleInfo info = filterBundle.getInfo();
+      final long buildTime = System.nanoTime() - bitmapConstructionStartNs;
+      log.debug("Filter partitioning (%sms):%s", TimeUnit.NANOSECONDS.toMillis(buildTime), info);
+    }
+    return filterBundle;
+  }
+
+  /**
+   * Returns the query-level {@link Filter} plus, if needed, a {@link RangeFilter} for
+   * {@link ColumnHolder#TIME_COLUMN_NAME}. The time filter is added if time order is {@link Order#NONE} and
+   * the provided {@link Interval} is not contained entirely within [minDataTimestamp, maxDataTimestamp].
+   */
+  @Nullable
+  private static Filter computeFilterWithIntervalIfNeeded(
+      final Order timeOrder,
+      final long minDataTimestamp,
+      final long maxDataTimestamp,
+      final Interval interval,
+      @Nullable final Filter filter
+  )
+  {
+    if (timeOrder == Order.NONE
+        && (minDataTimestamp < interval.getStartMillis() || maxDataTimestamp >= interval.getEndMillis())) {
+      final RangeFilter timeFilter = new RangeFilter(
+          ColumnHolder.TIME_COLUMN_NAME,
+          ColumnType.LONG,
+          minDataTimestamp < interval.getStartMillis() ? interval.getStartMillis() : null,
+          maxDataTimestamp >= interval.getEndMillis() ? interval.getEndMillis() : null,
+          false,
+          true,
+          null
       );
-      if (metrics != null) {
-        final long buildTime = System.nanoTime() - bitmapConstructionStartNs;
-        metrics.reportBitmapConstructionTime(buildTime);
-        final FilterBundle.BundleInfo info = filterBundle.getInfo();
-        metrics.filterBundle(info);
-        log.debug("Filter partitioning (%sms):%s", TimeUnit.NANOSECONDS.toMillis(buildTime), info);
-        if (filterBundle.getIndex() != null) {
-          metrics.reportPreFilteredRows(filterBundle.getIndex().getBitmap().size());
-        } else {
-          metrics.reportPreFilteredRows(0);
-        }
-      } else if (log.isDebugEnabled()) {
-        final FilterBundle.BundleInfo info = filterBundle.getInfo();
-        final long buildTime = System.nanoTime() - bitmapConstructionStartNs;
-        log.debug("Filter partitioning (%sms):%s", TimeUnit.NANOSECONDS.toMillis(buildTime), info);
+
+      if (filter == null) {
+        return timeFilter;
+      } else {
+        return new AndFilter(ImmutableList.of(filter, timeFilter));
       }
-      return filterBundle;
+    } else {
+      return filter;
     }
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
index f97f4a1bade..0060b276c4b 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
@@ -21,7 +21,10 @@ package org.apache.druid.segment;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.query.Order;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.column.BaseColumn;
 import org.apache.druid.segment.column.ColumnCapabilities;
@@ -60,13 +63,49 @@ public class QueryableIndexIndexableAdapter implements IndexableAdapter
   private final QueryableIndex input;
   private final ImmutableList<String> availableDimensions;
   private final Metadata metadata;
+  private final int timePositionForComparator;
 
   public QueryableIndexIndexableAdapter(QueryableIndex input)
   {
     this.input = input;
     numRows = input.getNumRows();
     availableDimensions = ImmutableList.copyOf(input.getAvailableDimensions());
+    if (availableDimensions.contains(ColumnHolder.TIME_COLUMN_NAME)) {
+      throw DruidException.defensive("Unexpectedly encountered dimension[%s]", ColumnHolder.TIME_COLUMN_NAME);
+    }
     this.metadata = input.getMetadata();
+
+    final List<OrderBy> inputOrdering = input.getOrdering();
+
+    int foundTimePosition = -1;
+    int i = 0;
+
+    // Some sort columns may not exist in the index, for example if they are omitted due to being 100% nulls.
+    // Locate the __time column in the sort order, skipping any nonexistent columns. This will be the position of
+    // the __time column within the dimension handlers.
+    for (final OrderBy orderBy : inputOrdering) {
+      final String columnName = orderBy.getColumnName();
+
+      if (orderBy.getOrder() != Order.ASCENDING) {
+        throw DruidException.defensive("Order[%s] for column[%s] is not supported", orderBy.getOrder(), columnName);
+      }
+
+      if (ColumnHolder.TIME_COLUMN_NAME.equals(columnName)) {
+        foundTimePosition = i;
+        break;
+      } else if (input.getDimensionHandlers().containsKey(columnName)) {
+        i++;
+      }
+    }
+
+    if (foundTimePosition >= 0) {
+      this.timePositionForComparator = foundTimePosition;
+    } else {
+      // Sort order is set, but does not contain __time. Indexable adapters involve all columns in TimeAndDimsPointer
+      // comparators, so we need to put the __time column somewhere. Put it immediately after the ones in the
+      // sort order.
+      this.timePositionForComparator = inputOrdering.size();
+    }
   }
 
   public QueryableIndex getQueryableIndex()
@@ -87,16 +126,23 @@ public class QueryableIndexIndexableAdapter implements IndexableAdapter
   }
 
   @Override
-  public List<String> getDimensionNames()
+  public List<String> getDimensionNames(final boolean includeTime)
   {
-    return availableDimensions;
+    if (includeTime) {
+      final List<String> retVal = new ArrayList<>(availableDimensions.size() + 1);
+      retVal.add(ColumnHolder.TIME_COLUMN_NAME);
+      retVal.addAll(availableDimensions);
+      return retVal;
+    } else {
+      return availableDimensions;
+    }
   }
 
   @Override
   public List<String> getMetricNames()
   {
     final Set<String> columns = Sets.newLinkedHashSet(input.getColumnNames());
-    final HashSet<String> dimensions = Sets.newHashSet(getDimensionNames());
+    final HashSet<String> dimensions = Sets.newHashSet(availableDimensions);
     return ImmutableList.copyOf(Sets.difference(columns, dimensions));
   }
 
@@ -262,7 +308,7 @@ public class QueryableIndexIndexableAdapter implements IndexableAdapter
 
       final ColumnSelectorFactory columnSelectorFactory = new QueryableIndexColumnSelectorFactory(
           VirtualColumns.EMPTY,
-          false,
+          timePositionForComparator == 0 ? Order.ASCENDING : Order.NONE,
           offset,
           columnCache
       );
@@ -292,6 +338,7 @@ public class QueryableIndexIndexableAdapter implements IndexableAdapter
 
       rowPointer = new RowPointer(
           rowTimestampSelector,
+          timePositionForComparator,
           rowDimensionValueSelectors,
           dimensionHandlers,
           rowMetricSelectors,
@@ -309,6 +356,7 @@ public class QueryableIndexIndexableAdapter implements IndexableAdapter
           .toArray(SettableColumnValueSelector[]::new);
       markedRowPointer = new TimeAndDimsPointer(
           markedTimestampSelector,
+          timePositionForComparator,
           markedDimensionValueSelectors,
           dimensionHandlers,
           markedMetricSelectors,
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexSegment.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexSegment.java
index b8d4d2d16cf..ccbe0475c33 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexSegment.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexSegment.java
@@ -31,6 +31,7 @@ import java.util.Map;
 import java.util.function.Function;
 
 /**
+ *
  */
 public class QueryableIndexSegment implements Segment
 {
@@ -39,12 +40,14 @@ public class QueryableIndexSegment implements Segment
 
   private final QueryableIndex index;
   private final QueryableIndexStorageAdapter storageAdapter;
+  private final TimeBoundaryInspector timeBoundaryInspector;
   private final SegmentId segmentId;
 
   public QueryableIndexSegment(QueryableIndex index, final SegmentId segmentId)
   {
     this.index = index;
     this.storageAdapter = new QueryableIndexStorageAdapter(index);
+    this.timeBoundaryInspector = QueryableIndexTimeBoundaryInspector.create(index);
     this.segmentId = segmentId;
   }
 
@@ -86,7 +89,14 @@ public class QueryableIndexSegment implements Segment
   {
     final Function<QueryableIndexSegment, ?> fn = AS_MAP.get(clazz);
     if (fn != null) {
-      return (T) fn.apply(this);
+      final T fnApply = (T) fn.apply(this);
+      if (fnApply != null) {
+        return fnApply;
+      }
+    }
+
+    if (TimeBoundaryInspector.class.equals(clazz)) {
+      return (T) timeBoundaryInspector;
     }
 
     return Segment.super.as(clazz);
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
index 7d860465cb6..e2c88071607 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
@@ -19,13 +19,11 @@
 
 package org.apache.druid.segment;
 
-import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.segment.column.BaseColumn;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnIndexSupplier;
 import org.apache.druid.segment.column.DictionaryEncodedColumn;
-import org.apache.druid.segment.column.NumericColumn;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex;
 import org.joda.time.DateTime;
@@ -46,12 +44,6 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
 
   private final QueryableIndex index;
 
-  @Nullable
-  private volatile DateTime minTime;
-
-  @Nullable
-  private volatile DateTime maxTime;
-
   public QueryableIndexStorageAdapter(QueryableIndex index)
   {
     this.index = index;
@@ -107,28 +99,6 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
     return index.getNumRows();
   }
 
-  @Override
-  public DateTime getMinTime()
-  {
-    if (minTime == null) {
-      // May be called a few times in parallel when first populating minTime, but this is benign, so allow it.
-      populateMinMaxTime();
-    }
-
-    return minTime;
-  }
-
-  @Override
-  public DateTime getMaxTime()
-  {
-    if (maxTime == null) {
-      // May be called a few times in parallel when first populating maxTime, but this is benign, so allow it.
-      populateMinMaxTime();
-    }
-
-    return maxTime;
-  }
-
   @Override
   @Nullable
   public Comparable getMinValue(String dimension)
@@ -183,14 +153,4 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
   {
     return index.getMetadata();
   }
-
-  private void populateMinMaxTime()
-  {
-    // Compute and cache minTime, maxTime.
-    final ColumnHolder columnHolder = index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME);
-    try (NumericColumn column = (NumericColumn) columnHolder.getColumn()) {
-      this.minTime = DateTimes.utc(column.getLongSingleValueRow(0));
-      this.maxTime = DateTimes.utc(column.getLongSingleValueRow(column.length() - 1));
-    }
-  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexTimeBoundaryInspector.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexTimeBoundaryInspector.java
new file mode 100644
index 00000000000..28292fc2439
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexTimeBoundaryInspector.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.query.Order;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.NumericColumn;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.joda.time.DateTime;
+
+/**
+ * {@link TimeBoundaryInspector} for {@link QueryableIndex} that are sorted by {@link ColumnHolder#TIME_COLUMN_NAME}.
+ */
+public class QueryableIndexTimeBoundaryInspector implements TimeBoundaryInspector
+{
+  private final QueryableIndex index;
+  private final boolean timeOrdered;
+
+  private QueryableIndexTimeBoundaryInspector(final QueryableIndex index, final boolean timeOrdered)
+  {
+    this.index = index;
+    this.timeOrdered = timeOrdered;
+  }
+
+  public static QueryableIndexTimeBoundaryInspector create(final QueryableIndex index)
+  {
+    return new QueryableIndexTimeBoundaryInspector(
+        index,
+        Cursors.getTimeOrdering(index.getOrdering()) == Order.ASCENDING
+    );
+  }
+
+  @MonotonicNonNull
+  private volatile DateTime minTime;
+
+  @MonotonicNonNull
+  private volatile DateTime maxTime;
+
+  @Override
+  public DateTime getMinTime()
+  {
+    if (minTime == null) {
+      // May be called a few times in parallel when first populating minTime, but this is benign, so allow it.
+      populateMinMaxTime();
+    }
+
+    return minTime;
+  }
+
+  @Override
+  public DateTime getMaxTime()
+  {
+    if (maxTime == null) {
+      // May be called a few times in parallel when first populating maxTime, but this is benign, so allow it.
+      populateMinMaxTime();
+    }
+
+    return maxTime;
+  }
+
+  @Override
+  public boolean isMinMaxExact()
+  {
+    return timeOrdered;
+  }
+
+  private void populateMinMaxTime()
+  {
+    if (timeOrdered) {
+      // Compute and cache minTime, maxTime.
+      final ColumnHolder columnHolder = index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME);
+      try (NumericColumn column = (NumericColumn) columnHolder.getColumn()) {
+        this.minTime = DateTimes.utc(column.getLongSingleValueRow(0));
+        this.maxTime = DateTimes.utc(column.getLongSingleValueRow(column.length() - 1));
+      }
+    } else {
+      // Use metadata. (Will be inexact.)
+      this.minTime = index.getDataInterval().getStart();
+      this.maxTime = index.getDataInterval().getEnd();
+    }
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/RowBasedStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/RowBasedStorageAdapter.java
index f4330709a44..eaed700f59c 100644
--- a/processing/src/main/java/org/apache/druid/segment/RowBasedStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/RowBasedStorageAdapter.java
@@ -34,7 +34,6 @@ import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.ListIndexed;
 import org.apache.druid.utils.CloseableUtils;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -57,7 +56,8 @@ public class RowBasedStorageAdapter<RowType> implements StorageAdapter
   public RowBasedStorageAdapter(
       final Sequence<RowType> rowSequence,
       final RowAdapter<RowType> rowAdapter,
-      final RowSignature rowSignature)
+      final RowSignature rowSignature
+  )
   {
     this.rowSequence = Preconditions.checkNotNull(rowSequence, "rowSequence");
     this.rowAdapter = Preconditions.checkNotNull(rowAdapter, "rowAdapter");
@@ -105,18 +105,6 @@ public class RowBasedStorageAdapter<RowType> implements StorageAdapter
     return DimensionDictionarySelector.CARDINALITY_UNKNOWN;
   }
 
-  @Override
-  public DateTime getMinTime()
-  {
-    return getInterval().getStart();
-  }
-
-  @Override
-  public DateTime getMaxTime()
-  {
-    return getInterval().getEnd().minus(1);
-  }
-
   @Nullable
   @Override
   public Comparable getMinValue(String column)
@@ -154,12 +142,6 @@ public class RowBasedStorageAdapter<RowType> implements StorageAdapter
     throw new UnsupportedOperationException("Cannot retrieve number of rows");
   }
 
-  @Override
-  public DateTime getMaxIngestedEventTime()
-  {
-    return getMaxTime();
-  }
-
   @Override
   public Metadata getMetadata()
   {
@@ -169,7 +151,9 @@ public class RowBasedStorageAdapter<RowType> implements StorageAdapter
   @Override
   public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    // It's in principle incorrect for sort order to be __time based here, but for historical reasons, we're keeping
+    // this in place for now. The handling of "interval" in "RowBasedCursor", which has been in place for some time,
+    // suggests we think the data is always sorted by time.
     final List<OrderBy> ordering;
     final boolean descending;
     if (Cursors.preferDescendingTimeOrdering(spec)) {
diff --git a/processing/src/main/java/org/apache/druid/segment/RowCombiningTimeAndDimsIterator.java b/processing/src/main/java/org/apache/druid/segment/RowCombiningTimeAndDimsIterator.java
index 5120ece6a95..919f53a41a0 100644
--- a/processing/src/main/java/org/apache/druid/segment/RowCombiningTimeAndDimsIterator.java
+++ b/processing/src/main/java/org/apache/druid/segment/RowCombiningTimeAndDimsIterator.java
@@ -128,6 +128,7 @@ final class RowCombiningTimeAndDimsIterator implements TimeAndDimsIterator
           if (markedRowPointer != null) {
             return new TimeAndDimsPointer(
                 markedRowPointer.timestampSelector,
+                markedRowPointer.timePosition,
                 markedRowPointer.dimensionSelectors,
                 markedRowPointer.getDimensionHandlers(),
                 combinedMetricSelectors,
diff --git a/processing/src/main/java/org/apache/druid/segment/RowPointer.java b/processing/src/main/java/org/apache/druid/segment/RowPointer.java
index dcb7ab6b204..7a67b748287 100644
--- a/processing/src/main/java/org/apache/druid/segment/RowPointer.java
+++ b/processing/src/main/java/org/apache/druid/segment/RowPointer.java
@@ -41,6 +41,7 @@ public final class RowPointer extends TimeAndDimsPointer
 
   public RowPointer(
       ColumnValueSelector timestampSelector,
+      int timePosition,
       ColumnValueSelector[] dimensionSelectors,
       List<DimensionHandler> dimensionHandlers,
       ColumnValueSelector[] metricSelectors,
@@ -48,7 +49,7 @@ public final class RowPointer extends TimeAndDimsPointer
       IntSupplier rowNumPointer
   )
   {
-    super(timestampSelector, dimensionSelectors, dimensionHandlers, metricSelectors, metricNames);
+    super(timestampSelector, timePosition, dimensionSelectors, dimensionHandlers, metricSelectors, metricNames);
     this.rowNumPointer = rowNumPointer;
   }
 
@@ -62,6 +63,7 @@ public final class RowPointer extends TimeAndDimsPointer
   {
     return new RowPointer(
         timestampSelector,
+        timePosition,
         newDimensionSelectors,
         getDimensionHandlers(),
         metricSelectors,
diff --git a/processing/src/main/java/org/apache/druid/segment/Segment.java b/processing/src/main/java/org/apache/druid/segment/Segment.java
index fafd3f500ac..44be82a56ee 100644
--- a/processing/src/main/java/org/apache/druid/segment/Segment.java
+++ b/processing/src/main/java/org/apache/druid/segment/Segment.java
@@ -20,6 +20,8 @@
 package org.apache.druid.segment;
 
 import org.apache.druid.guice.annotations.PublicApi;
+import org.apache.druid.query.datasourcemetadata.DataSourceMetadataResultValue;
+import org.apache.druid.segment.join.table.IndexedTable;
 import org.apache.druid.timeline.SegmentId;
 import org.joda.time.Interval;
 
@@ -36,11 +38,14 @@ import java.io.Closeable;
 public interface Segment extends Closeable
 {
   SegmentId getId();
+
   Interval getDataInterval();
+
   @Nullable
   QueryableIndex asQueryableIndex();
+
   StorageAdapter asStorageAdapter();
-  
+
   /**
    * Request an implementation of a particular interface.
    *
@@ -49,14 +54,22 @@ public interface Segment extends Closeable
    * expected to be requested by callers that have specific knowledge of extra features provided by specific
    * segment types. For example, an extension might provide a custom Segment type that can offer both
    * StorageAdapter and some new interface. That extension can also offer a Query that uses that new interface.
-   * 
-   * Implementations which accept classes other than {@link QueryableIndex} or {@link StorageAdapter} are limited 
-   * to using those classes within the extension. This means that one extension cannot rely on the `Segment.as` 
+   *
+   * Implementations which accept classes other than {@link QueryableIndex} or {@link StorageAdapter} are limited
+   * to using those classes within the extension. This means that one extension cannot rely on the `Segment.as`
    * behavior of another extension.
    *
    * @param clazz desired interface
-   * @param <T> desired interface
+   * @param <T>   desired interface
+   *
    * @return instance of clazz, or null if the interface is not supported by this segment
+   *
+   * @see StorageAdapter storage adapter for queries. Never null.
+   * @see QueryableIndex index object, if this is a memory-mapped regular segment.
+   * @see IndexedTable table object, if this is a joinable indexed table.
+   * @see TimeBoundaryInspector inspector for min/max timestamps, if supported by this segment.
+   * @see MaxIngestedEventTimeInspector inspector for {@link DataSourceMetadataResultValue#getMaxIngestedEventTime()}
+   * @see CloseableShapeshifter stepping stone to {@link org.apache.druid.query.rowsandcols.RowsAndColumns}.
    */
   @SuppressWarnings({"unused", "unchecked"})
   @Nullable
diff --git a/processing/src/main/java/org/apache/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/org/apache/druid/segment/SimpleQueryableIndex.java
index 013a634fdc4..507737a2c5b 100644
--- a/processing/src/main/java/org/apache/druid/segment/SimpleQueryableIndex.java
+++ b/processing/src/main/java/org/apache/druid/segment/SimpleQueryableIndex.java
@@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import org.apache.druid.collections.bitmap.BitmapFactory;
 import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.data.Indexed;
 import org.joda.time.Interval;
@@ -102,6 +103,18 @@ public abstract class SimpleQueryableIndex implements QueryableIndex
     return availableDimensions;
   }
 
+  @Override
+  public List<OrderBy> getOrdering()
+  {
+    final Metadata metadata = getMetadata();
+    if (metadata != null && metadata.getOrdering() != null) {
+      return metadata.getOrdering();
+    } else {
+      // When sort order isn't set in metadata.drd, assume the segment is sorted by __time.
+      return Cursors.ascendingTimeOrder();
+    }
+  }
+
   @Override
   public BitmapFactory getBitmapFactoryForDimensions()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/SingleScanTimeDimensionSelector.java b/processing/src/main/java/org/apache/druid/segment/SingleScanTimeDimensionSelector.java
index 681738360c5..136c4ef271e 100644
--- a/processing/src/main/java/org/apache/druid/segment/SingleScanTimeDimensionSelector.java
+++ b/processing/src/main/java/org/apache/druid/segment/SingleScanTimeDimensionSelector.java
@@ -20,6 +20,8 @@
 package org.apache.druid.segment;
 
 import com.google.common.base.Preconditions;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.extraction.ExtractionFn;
 import org.apache.druid.query.filter.DruidObjectPredicate;
 import org.apache.druid.query.filter.DruidPredicateFactory;
@@ -43,7 +45,7 @@ public class SingleScanTimeDimensionSelector implements DimensionSelector
 {
   private final ExtractionFn extractionFn;
   private final BaseLongColumnValueSelector selector;
-  private final boolean descending;
+  private final Order timeOrder;
 
   private final List<String> timeValues = new ArrayList<>();
   private final SingleIndexedInt row = new SingleIndexedInt();
@@ -56,13 +58,17 @@ public class SingleScanTimeDimensionSelector implements DimensionSelector
   public SingleScanTimeDimensionSelector(
       BaseLongColumnValueSelector selector,
       @Nullable ExtractionFn extractionFn,
-      boolean descending
+      Order timeOrder
   )
   {
     Preconditions.checkNotNull(extractionFn, "time dimension must provide an extraction function");
     this.extractionFn = extractionFn;
     this.selector = selector;
-    this.descending = descending;
+    this.timeOrder = timeOrder;
+
+    if (timeOrder == Order.NONE) {
+      throw DruidException.defensive("Cannot use timeOrder[%s]", timeOrder);
+    }
   }
 
   @Override
@@ -130,7 +136,7 @@ public class SingleScanTimeDimensionSelector implements DimensionSelector
       // we can also avoid creating a dimension value and corresponding index
       // and use the current one
     } else if (timestamp != currentTimestamp) {
-      if (descending ? timestamp > currentTimestamp : timestamp < currentTimestamp) {
+      if (timeOrder == Order.ASCENDING ? timestamp < currentTimestamp : timestamp > currentTimestamp) {
         // re-using this selector for multiple scans would cause the same rows to return different IDs
         // we might want to re-visit if we ever need to do multiple scans with this dimension selector
         throw new IllegalStateException("cannot re-use time dimension selector for multiple scans");
@@ -198,6 +204,6 @@ public class SingleScanTimeDimensionSelector implements DimensionSelector
   {
     inspector.visit("selector", selector);
     inspector.visit("extractionFn", extractionFn);
-    inspector.visit("descending", descending);
+    inspector.visit("timeOrder", timeOrder);
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
index fba947f73d2..3ad6c930002 100644
--- a/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
@@ -20,10 +20,12 @@
 package org.apache.druid.segment;
 
 import com.google.common.collect.Iterables;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.guice.annotations.PublicApi;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.vector.VectorCursor;
@@ -35,6 +37,7 @@ import java.util.List;
 import java.util.Optional;
 
 /**
+ *
  */
 @PublicApi
 public interface StorageAdapter extends CursorFactory, ColumnInspector, CursorHolderFactory
@@ -49,7 +52,8 @@ public interface StorageAdapter extends CursorFactory, ColumnInspector, CursorHo
   @Override
   default CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
-    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
+    // For backwards compatibility, the default implementation assumes the underlying rows are sorted by __time.
+    // Built-in implementations of StorageAdapter must override this method.
     final List<OrderBy> ordering;
     final boolean descending;
     if (Cursors.preferDescendingTimeOrdering(spec)) {
@@ -67,7 +71,7 @@ public interface StorageAdapter extends CursorFactory, ColumnInspector, CursorHo
         return StorageAdapter.this.canVectorize(
             spec.getFilter(),
             spec.getVirtualColumns(),
-            Cursors.preferDescendingTimeOrdering(spec)
+            descending
         );
       }
 
@@ -115,7 +119,15 @@ public interface StorageAdapter extends CursorFactory, ColumnInspector, CursorHo
   }
 
   Interval getInterval();
+
+  /**
+   * Returns the names of all dimension columns, not including {@link ColumnHolder#TIME_COLUMN_NAME}.
+   */
   Indexed<String> getAvailableDimensions();
+
+  /**
+   * Returns the names of all metric columns.
+   */
   Iterable<String> getAvailableMetrics();
 
   /**
@@ -147,28 +159,26 @@ public interface StorageAdapter extends CursorFactory, ColumnInspector, CursorHo
   int getDimensionCardinality(String column);
 
   /**
-   * Metadata-only operation that returns a lower bound on
-   * {@link org.apache.druid.segment.column.ColumnHolder#TIME_COLUMN_NAME} values for this adapter. May be earlier than
-   * the actual minimum data timestamp.
-   *
-   * For {@link QueryableIndexStorageAdapter} and {@link org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter}
-   * specifically, which back regular tables (i.e. {@link org.apache.druid.query.TableDataSource}), this method
-   * contract is tighter: it does return the actual minimum data timestamp. This fact is leveraged by
-   * {@link org.apache.druid.query.timeboundary.TimeBoundaryQuery} to return results using metadata only.
+   * Use {@link TimeBoundaryInspector#getMinTime()} instead.
    */
-  DateTime getMinTime();
+  @Deprecated
+  default DateTime getMinTime()
+  {
+    throw DruidException.defensive(
+        "getMinTime is no longer supported, use Segment.as(MinMaxValueInspector.class) instead"
+    );
+  }
 
   /**
-   * Metadata-only operation that returns an upper bound on
-   * {@link org.apache.druid.segment.column.ColumnHolder#TIME_COLUMN_NAME} values for this adapter. May be later than
-   * the actual maximum data timestamp.
-   *
-   * For {@link QueryableIndexStorageAdapter} and {@link org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter}
-   * specifically, which back regular tables (i.e. {@link org.apache.druid.query.TableDataSource}), this method
-   * contract is tighter: it does return the actual maximum data timestamp. This fact is leveraged by
-   * {@link org.apache.druid.query.timeboundary.TimeBoundaryQuery} to return results using metadata only.
+   * Use {@link TimeBoundaryInspector#getMaxTime()} instead.
    */
-  DateTime getMaxTime();
+  @Deprecated
+  default DateTime getMaxTime()
+  {
+    throw DruidException.defensive(
+        "getMaxTime is no longer supported, use Segment.as(MinMaxValueInspector.class) instead"
+    );
+  }
 
   /**
    * Returns the minimum value of the provided column, if known through an index, dictionary, or cache. Returns null
@@ -203,7 +213,17 @@ public interface StorageAdapter extends CursorFactory, ColumnInspector, CursorHo
   ColumnCapabilities getColumnCapabilities(String column);
 
   int getNumRows();
-  DateTime getMaxIngestedEventTime();
+
+  /**
+   * Use {@link MaxIngestedEventTimeInspector#getMaxIngestedEventTime()} instead.
+   */
+  @Deprecated
+  default DateTime getMaxIngestedEventTime()
+  {
+    throw DruidException.defensive(
+        "getMaxIngestedEventTime is no longer supported, use Segment.as(MaxIngestedEventTimeInspector.class) instead"
+    );
+  }
 
   @Nullable
   Metadata getMetadata();
diff --git a/processing/src/main/java/org/apache/druid/segment/TimeAndDimsPointer.java b/processing/src/main/java/org/apache/druid/segment/TimeAndDimsPointer.java
index 56eeba56655..4472ef099ae 100644
--- a/processing/src/main/java/org/apache/druid/segment/TimeAndDimsPointer.java
+++ b/processing/src/main/java/org/apache/druid/segment/TimeAndDimsPointer.java
@@ -20,7 +20,9 @@
 package org.apache.druid.segment;
 
 import com.google.common.base.Preconditions;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.segment.column.ColumnHolder;
 
 import javax.annotation.Nonnull;
 import java.util.Arrays;
@@ -49,6 +51,12 @@ import java.util.stream.Collectors;
 public class TimeAndDimsPointer implements Comparable<TimeAndDimsPointer>
 {
   final ColumnValueSelector timestampSelector;
+  /**
+   * Position of {@link ColumnHolder#TIME_COLUMN_NAME} in the sort order, relative to elements of
+   * {@link #dimensionSelectors}. For example, for the sort order [x, __time, y], {@link #dimensionSelectors}
+   * contains [x, y] and timePosition is 1.
+   */
+  final int timePosition;
   /**
    * This collection of dimension selectors is stored as array rather than List in order to minimize indirection in hot
    * spots, in particular in {@link #compareTo}.
@@ -76,6 +84,7 @@ public class TimeAndDimsPointer implements Comparable<TimeAndDimsPointer>
    */
   TimeAndDimsPointer(
       ColumnValueSelector timestampSelector,
+      int timePosition,
       ColumnValueSelector[] dimensionSelectors,
       List<DimensionHandler> dimensionHandlers,
       ColumnValueSelector[] metricSelectors,
@@ -83,6 +92,13 @@ public class TimeAndDimsPointer implements Comparable<TimeAndDimsPointer>
   )
   {
     this.timestampSelector = timestampSelector;
+    this.timePosition = timePosition;
+    Preconditions.checkArgument(
+        timePosition >= 0 && timePosition <= dimensionSelectors.length,
+        "timePosition[%s] is out of range 0 .. [%s]",
+        timePosition,
+        dimensionSelectors.length
+    );
     Preconditions.checkArgument(dimensionSelectors.length == dimensionHandlers.size());
     this.dimensionSelectors = dimensionSelectors;
     this.dimensionHandlers = dimensionHandlers;
@@ -131,10 +147,23 @@ public class TimeAndDimsPointer implements Comparable<TimeAndDimsPointer>
     return metricNames;
   }
 
+  /**
+   * Replace {@link #dimensionSelectors} with a new set of selectors. The new selectors must be the same length.
+   * Time position ({@link #timePosition}) is unchanged.
+   */
   TimeAndDimsPointer withDimensionSelectors(ColumnValueSelector[] newDimensionSelectors)
   {
+    if (dimensionSelectors.length != newDimensionSelectors.length) {
+      throw DruidException.defensive(
+          "Lengths of dimensionSelectors[%d] and newDimensionSelectors[%d] do not match",
+          dimensionSelectors.length,
+          newDimensionSelectors.length
+      );
+    }
+
     return new TimeAndDimsPointer(
         timestampSelector,
+        timePosition,
         newDimensionSelectors,
         dimensionHandlers,
         metricSelectors,
@@ -148,19 +177,23 @@ public class TimeAndDimsPointer implements Comparable<TimeAndDimsPointer>
   @Override
   public int compareTo(@Nonnull TimeAndDimsPointer rhs)
   {
-    long timestamp = getTimestamp();
-    long rhsTimestamp = rhs.getTimestamp();
-    int timestampDiff = Long.compare(timestamp, rhsTimestamp);
-    if (timestampDiff != 0) {
-      return timestampDiff;
-    }
-    for (int dimIndex = 0; dimIndex < dimensionSelectors.length; dimIndex++) {
-      int dimDiff = dimensionSelectorComparators[dimIndex].compare(
-          dimensionSelectors[dimIndex],
-          rhs.dimensionSelectors[dimIndex]
-      );
-      if (dimDiff != 0) {
-        return dimDiff;
+    for (int index = 0; index < (dimensionSelectors.length + 1); index++) {
+      if (index == timePosition) {
+        final long timestamp = getTimestamp();
+        final long rhsTimestamp = rhs.getTimestamp();
+        final int timestampDiff = Long.compare(timestamp, rhsTimestamp);
+        if (timestampDiff != 0) {
+          return timestampDiff;
+        }
+      } else {
+        final int dimIndex = index < timePosition ? index : index - 1;
+        final int dimDiff = dimensionSelectorComparators[dimIndex].compare(
+            dimensionSelectors[dimIndex],
+            rhs.dimensionSelectors[dimIndex]
+        );
+        if (dimDiff != 0) {
+          return dimDiff;
+        }
       }
     }
     return 0;
diff --git a/processing/src/main/java/org/apache/druid/segment/TimeBoundaryInspector.java b/processing/src/main/java/org/apache/druid/segment/TimeBoundaryInspector.java
new file mode 100644
index 00000000000..00d379696d4
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/TimeBoundaryInspector.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import org.apache.druid.segment.column.ColumnHolder;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+
+/**
+ * Returns min/max values of {@link ColumnHolder#TIME_COLUMN_NAME} for a segment. Typically retrieved from
+ * {@link Segment#as(Class)}.
+ *
+ * Only offered by segments that are able to provide more detailed min/max values via metadata inspection than are
+ * available from {@link Segment#getDataInterval()}.
+ */
+public interface TimeBoundaryInspector
+{
+  /**
+   * Lower bound on {@link ColumnHolder#TIME_COLUMN_NAME}. Matches the lowest timestamp in the dataset
+   * if {@link #isMinMaxExact()}.
+   */
+  DateTime getMinTime();
+
+  /**
+   * Upper bound on {@link ColumnHolder#TIME_COLUMN_NAME}. Strict if {@link #isMinMaxExact()}. Matches the highest
+   * timestamp in the dataset if {@link #isMinMaxExact()}.
+   */
+  DateTime getMaxTime();
+
+  /**
+   * Smallest interval that contains {@link #getMinTime()} and {@link #getMaxTime()}. The endpoint is one millisecond
+   * higher than {@link #getMaxTime()}.
+   */
+  default Interval getMinMaxInterval()
+  {
+    return new Interval(getMinTime(), getMaxTime().plus(1));
+  }
+
+  /**
+   * Whether the lower and upper bounds returned by {@link #getMinTime()} and {@link #getMaxTime()} are actually
+   * found in the dataset. If true, the bounds are strict and can be used substitutes for aggregations
+   * {@code MIN(__time)} and {@code MAX(__time)}.
+   */
+  boolean isMinMaxExact();
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
index a9f215892d1..435771b4780 100644
--- a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
@@ -49,7 +49,6 @@ import org.apache.druid.segment.filter.SelectorFilter;
 import org.apache.druid.segment.join.PostJoinCursor;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
 import org.apache.druid.utils.CloseableUtils;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -150,11 +149,10 @@ public class UnnestStorageAdapter implements StorageAdapter
         );
       }
 
-      @Nullable
       @Override
       public List<OrderBy> getOrdering()
       {
-        return cursorHolderSupplier.get().getOrdering();
+        return computeOrdering(cursorHolderSupplier.get().getOrdering());
       }
 
       @Override
@@ -204,18 +202,6 @@ public class UnnestStorageAdapter implements StorageAdapter
     return DimensionDictionarySelector.CARDINALITY_UNKNOWN;
   }
 
-  @Override
-  public DateTime getMinTime()
-  {
-    return baseAdapter.getMinTime();
-  }
-
-  @Override
-  public DateTime getMaxTime()
-  {
-    return baseAdapter.getMaxTime();
-  }
-
   @Nullable
   @Override
   public Comparable getMinValue(String column)
@@ -255,12 +241,6 @@ public class UnnestStorageAdapter implements StorageAdapter
     return 0;
   }
 
-  @Override
-  public DateTime getMaxIngestedEventTime()
-  {
-    return baseAdapter.getMaxIngestedEventTime();
-  }
-
   @Nullable
   @Override
   public Metadata getMetadata()
@@ -579,6 +559,23 @@ public class UnnestStorageAdapter implements StorageAdapter
     }
   }
 
+  /**
+   * Computes ordering of a join {@link CursorHolder} based on the ordering of an underlying {@link CursorHolder}.
+   */
+  private List<OrderBy> computeOrdering(final List<OrderBy> baseOrdering)
+  {
+    // Sorted the same way as the base segment, unless the unnested column shadows one of the base columns.
+    int limit = 0;
+    for (; limit < baseOrdering.size(); limit++) {
+      final String columnName = baseOrdering.get(limit).getColumnName();
+      if (columnName.equals(outputColumnName) || columnName.equals(unnestColumn.getOutputName())) {
+        break;
+      }
+    }
+
+    return limit == baseOrdering.size() ? baseOrdering : baseOrdering.subList(0, limit);
+  }
+
   /**
    * Computes the capabilities of {@link #outputColumnName}, after unnesting.
    */
diff --git a/processing/src/main/java/org/apache/druid/segment/WrappedSegmentReference.java b/processing/src/main/java/org/apache/druid/segment/WrappedSegmentReference.java
index a327ff6f1f1..19db47067b5 100644
--- a/processing/src/main/java/org/apache/druid/segment/WrappedSegmentReference.java
+++ b/processing/src/main/java/org/apache/druid/segment/WrappedSegmentReference.java
@@ -23,6 +23,7 @@ import org.apache.druid.query.FilteredDataSource;
 import org.apache.druid.timeline.SegmentId;
 import org.joda.time.Interval;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.Closeable;
 import java.io.IOException;
@@ -79,6 +80,17 @@ public class WrappedSegmentReference implements SegmentReference
     return storageAdapterWrapperFunction.apply(delegate.asStorageAdapter());
   }
 
+  @Nullable
+  @Override
+  public <T> T as(@Nonnull Class<T> clazz)
+  {
+    if (TimeBoundaryInspector.class.equals(clazz)) {
+      return (T) WrappedTimeBoundaryInspector.create(delegate.as(TimeBoundaryInspector.class));
+    } else {
+      return SegmentReference.super.as(clazz);
+    }
+  }
+
   @Override
   public void close() throws IOException
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/WrappedTimeBoundaryInspector.java b/processing/src/main/java/org/apache/druid/segment/WrappedTimeBoundaryInspector.java
new file mode 100644
index 00000000000..4b2a21eb11b
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/WrappedTimeBoundaryInspector.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import com.google.common.base.Preconditions;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.joda.time.DateTime;
+
+import javax.annotation.Nullable;
+
+/**
+ * Wrapper for {@link TimeBoundaryInspector} used by {@link Segment} implementations that may filter out rows
+ * from an underlying segment, but do not modify {@link ColumnHolder#TIME_COLUMN_NAME}.
+ */
+public class WrappedTimeBoundaryInspector implements TimeBoundaryInspector
+{
+  private final TimeBoundaryInspector delegate;
+
+  private WrappedTimeBoundaryInspector(final TimeBoundaryInspector delegate)
+  {
+    this.delegate = Preconditions.checkNotNull(delegate, "delegate");
+  }
+
+  @Nullable
+  public static WrappedTimeBoundaryInspector create(@Nullable final TimeBoundaryInspector delegate)
+  {
+    if (delegate != null) {
+      return new WrappedTimeBoundaryInspector(delegate);
+    } else {
+      return null;
+    }
+  }
+
+  @Override
+  public DateTime getMinTime()
+  {
+    return delegate.getMinTime();
+  }
+
+  @Override
+  public DateTime getMaxTime()
+  {
+    return delegate.getMaxTime();
+  }
+
+  @Override
+  public boolean isMinMaxExact()
+  {
+    // Always false, because rows may be filtered out.
+    return false;
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java
index 5265f4f2c44..cfcc99c10e6 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java
@@ -44,6 +44,7 @@ import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.parsers.ParseException;
 import org.apache.druid.java.util.common.parsers.UnparseableColumnsParseException;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.PostAggregator;
 import org.apache.druid.query.dimension.DimensionSpec;
@@ -93,6 +94,7 @@ import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
 
 /**
  * In-memory, row-based data structure used to hold data during ingestion. Realtime tasks query this index using
@@ -227,6 +229,9 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
   private final List<Function<InputRow, InputRow>> rowTransformers;
   private final VirtualColumns virtualColumns;
   private final AggregatorFactory[] metrics;
+  /**
+   * Metadata to be persisted along with this index, when it is eventually persisted.
+   */
   private final Metadata metadata;
   protected final boolean preserveExistingMetrics;
 
@@ -234,6 +239,12 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
 
   private final DimensionsSpec dimensionsSpec;
   private final Map<String, DimensionDesc> dimensionDescs;
+  /**
+   * Position of {@link ColumnHolder#TIME_COLUMN_NAME} in the sort order, relative to elements of
+   * {@link #dimensionDescs}. For example, for the sort order [x, __time, y], dimensionDescs contains [x, y] and
+   * timePosition is 1.
+   */
+  protected final int timePosition;
   private final List<DimensionDesc> dimensionDescsList;
   // dimension capabilities are provided by the indexers
   private final Map<String, ColumnCapabilities> timeAndMetricsColumnCapabilities;
@@ -280,13 +291,6 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
     this.timeAndMetricsColumnFormats = new HashMap<>();
     this.metricDescs = Maps.newLinkedHashMap();
     this.dimensionDescs = Maps.newLinkedHashMap();
-    this.metadata = new Metadata(
-        null,
-        getCombiningAggregators(metrics),
-        incrementalIndexSchema.getTimestampSpec(),
-        this.queryGranularity,
-        this.rollup
-    );
 
     initAggs(metrics, inputRowHolder);
 
@@ -319,8 +323,24 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
     this.dimensionsSpec = incrementalIndexSchema.getDimensionsSpec();
 
     this.dimensionDescsList = new ArrayList<>();
-    for (DimensionSchema dimSchema : dimensionsSpec.getDimensions()) {
-      addNewDimension(dimSchema.getName(), dimSchema.getDimensionHandler());
+
+    int foundTimePosition = -1;
+    final List<DimensionSchema> dimSchemas = dimensionsSpec.getDimensions();
+    for (int i = 0; i < dimSchemas.size(); i++) {
+      final DimensionSchema dimSchema = dimSchemas.get(i);
+      if (ColumnHolder.TIME_COLUMN_NAME.equals(dimSchema.getName())) {
+        foundTimePosition = i;
+      } else {
+        addNewDimension(dimSchema.getName(), dimSchema.getDimensionHandler());
+      }
+    }
+
+    if (foundTimePosition == -1) {
+      // __time not found: that means it either goes at the end, or the beginning, based on
+      // forceSegmentSortByTime.
+      this.timePosition = dimensionsSpec.isForceSegmentSortByTime() ? 0 : dimensionDescsList.size();
+    } else {
+      this.timePosition = foundTimePosition;
     }
 
     //__time capabilities
@@ -334,6 +354,16 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
     if (!spatialDimensions.isEmpty()) {
       this.rowTransformers.add(new SpatialDimensionRowTransformer(spatialDimensions));
     }
+
+    // Set metadata last, so dimensionOrder is populated
+    this.metadata = new Metadata(
+        null,
+        getCombiningAggregators(metrics),
+        incrementalIndexSchema.getTimestampSpec(),
+        this.queryGranularity,
+        this.rollup,
+        getDimensionOrder().stream().map(OrderBy::ascending).collect(Collectors.toList())
+    );
   }
 
   public abstract FactsHolder getFacts();
@@ -582,7 +612,7 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
       // first, process dimension values present in the row
       dims = new Object[dimensionDescs.size()];
       for (String dimension : rowDimensions) {
-        if (Strings.isNullOrEmpty(dimension)) {
+        if (Strings.isNullOrEmpty(dimension) || ColumnHolder.TIME_COLUMN_NAME.equals(dimension)) {
           continue;
         }
         boolean wasNewDim = false;
@@ -789,13 +819,38 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
     return dimensionsSpec;
   }
 
-  public List<String> getDimensionNames()
+  /**
+   * Returns names of dimension columns.
+   *
+   * @param includeTime whether to include {@link ColumnHolder#TIME_COLUMN_NAME}.
+   */
+  public List<String> getDimensionNames(final boolean includeTime)
   {
     synchronized (dimensionDescs) {
-      return ImmutableList.copyOf(dimensionDescs.keySet());
+      if (includeTime) {
+        final ImmutableList.Builder<String> listBuilder =
+            ImmutableList.builderWithExpectedSize(dimensionDescs.size() + 1);
+        int i = 0;
+        if (i == timePosition) {
+          listBuilder.add(ColumnHolder.TIME_COLUMN_NAME);
+        }
+        for (String dimName : dimensionDescs.keySet()) {
+          listBuilder.add(dimName);
+          i++;
+          if (i == timePosition) {
+            listBuilder.add(ColumnHolder.TIME_COLUMN_NAME);
+          }
+        }
+        return listBuilder.build();
+      } else {
+        return ImmutableList.copyOf(dimensionDescs.keySet());
+      }
     }
   }
 
+  /**
+   * Returns a descriptor for each dimension. Does not inclue {@link ColumnHolder#TIME_COLUMN_NAME}.
+   */
   public List<DimensionDesc> getDimensions()
   {
     synchronized (dimensionDescs) {
@@ -803,6 +858,9 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
     }
   }
 
+  /**
+   * Returns the descriptor for a particular dimension.
+   */
   @Nullable
   public DimensionDesc getDimension(String dimension)
   {
@@ -859,11 +917,12 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
     return dimSpec == null ? null : dimSpec.getIndex();
   }
 
+  /**
+   * Returns names of time and dimension columns, in persist sort order. Includes {@link ColumnHolder#TIME_COLUMN_NAME}.
+   */
   public List<String> getDimensionOrder()
   {
-    synchronized (dimensionDescs) {
-      return ImmutableList.copyOf(dimensionDescs.keySet());
-    }
+    return getDimensionNames(true);
   }
 
   public static ColumnCapabilitiesImpl makeDefaultCapabilitiesFromValueType(ColumnType type)
@@ -887,19 +946,23 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
    * Currently called to initialize IncrementalIndex dimension order during index creation
    * Index dimension ordering could be changed to initialize from DimensionsSpec after resolution of
    * https://github.com/apache/druid/issues/2011
+   *
+   * @param oldDimensionOrder dimension order to initialize
+   * @param oldColumnFormats  formats for the dimensions
    */
   public void loadDimensionIterable(
       Iterable<String> oldDimensionOrder,
-      Map<String, ColumnFormat> oldColumnCapabilities
+      Map<String, ColumnFormat> oldColumnFormats
   )
   {
     synchronized (dimensionDescs) {
-      if (!dimensionDescs.isEmpty()) {
-        throw new ISE("Cannot load dimension order when existing order[%s] is not empty.", dimensionDescs.keySet());
+      if (size() != 0) {
+        throw new ISE("Cannot load dimension order[%s] when existing index is not empty.", dimensionDescs.keySet());
       }
       for (String dim : oldDimensionOrder) {
-        if (dimensionDescs.get(dim) == null) {
-          ColumnFormat format = oldColumnCapabilities.get(dim);
+        // Skip __time; its position is solely based on configuration at index creation time.
+        if (!ColumnHolder.TIME_COLUMN_NAME.equals(dim) && dimensionDescs.get(dim) == null) {
+          ColumnFormat format = oldColumnFormats.get(dim);
           addNewDimension(dim, format.getColumnHandler(dim));
         }
       }
@@ -925,9 +988,12 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
     return ImmutableList.copyOf(metricDescs.keySet());
   }
 
+  /**
+   * Returns all column names, including {@link ColumnHolder#TIME_COLUMN_NAME}.
+   */
   public List<String> getColumnNames()
   {
-    List<String> columnNames = new ArrayList<>(getDimensionNames());
+    List<String> columnNames = new ArrayList<>(getDimensionNames(true));
     columnNames.addAll(getMetricNames());
     return columnNames;
   }
@@ -1068,33 +1134,48 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
 
   protected final Comparator<IncrementalIndexRow> dimsComparator()
   {
-    return new IncrementalIndexRowComparator(dimensionDescsList);
+    return new IncrementalIndexRowComparator(timePosition, dimensionDescsList);
   }
 
   @VisibleForTesting
   static final class IncrementalIndexRowComparator implements Comparator<IncrementalIndexRow>
   {
-    private List<DimensionDesc> dimensionDescs;
+    /**
+     * Position of {@link ColumnHolder#TIME_COLUMN_NAME} in the sort order.
+     */
+    private final int timePosition;
+    private final List<DimensionDesc> dimensionDescs;
 
-    public IncrementalIndexRowComparator(List<DimensionDesc> dimDescs)
+    public IncrementalIndexRowComparator(int timePosition, List<DimensionDesc> dimDescs)
     {
+      this.timePosition = timePosition;
       this.dimensionDescs = dimDescs;
     }
 
     @Override
     public int compare(IncrementalIndexRow lhs, IncrementalIndexRow rhs)
     {
-      int retVal = Longs.compare(lhs.timestamp, rhs.timestamp);
-      int numComparisons = Math.min(lhs.dims.length, rhs.dims.length);
+      int retVal = 0;
 
-      int index = 0;
-      while (retVal == 0 && index < numComparisons) {
-        final Object lhsIdxs = lhs.dims[index];
-        final Object rhsIdxs = rhs.dims[index];
+      // Number of dimension comparisons, not counting __time.
+      int numDimComparisons = Math.min(lhs.dims.length, rhs.dims.length);
+
+      int dimIndex = 0;
+      while (retVal == 0 && dimIndex < numDimComparisons) {
+        if (dimIndex == timePosition) {
+          retVal = Longs.compare(lhs.timestamp, rhs.timestamp);
+
+          if (retVal != 0) {
+            break;
+          }
+        }
+
+        final Object lhsIdxs = lhs.dims[dimIndex];
+        final Object rhsIdxs = rhs.dims[dimIndex];
 
         if (lhsIdxs == null) {
           if (rhsIdxs == null) {
-            ++index;
+            ++dimIndex;
             continue;
           }
           return -1;
@@ -1104,9 +1185,13 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
           return 1;
         }
 
-        final DimensionIndexer indexer = dimensionDescs.get(index).getIndexer();
+        final DimensionIndexer indexer = dimensionDescs.get(dimIndex).getIndexer();
         retVal = indexer.compareUnsortedEncodedKeyComponents(lhsIdxs, rhsIdxs);
-        ++index;
+        ++dimIndex;
+      }
+
+      if (retVal == 0 && dimIndex == numDimComparisons && timePosition >= numDimComparisons) {
+        retVal = Longs.compare(lhs.timestamp, rhs.timestamp);
       }
 
       if (retVal == 0) {
@@ -1115,7 +1200,7 @@ public abstract class IncrementalIndex implements Iterable<Row>, Closeable, Colu
           return 0;
         }
         Object[] largerDims = lengthDiff > 0 ? lhs.dims : rhs.dims;
-        return allNull(largerDims, numComparisons) ? 0 : lengthDiff;
+        return allNull(largerDims, numDimComparisons) ? 0 : lengthDiff;
       }
 
       return retVal;
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java
index a61ff334bb9..aae1c133575 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java
@@ -116,9 +116,9 @@ public class IncrementalIndexAdapter implements IndexableAdapter
   }
 
   @Override
-  public List<String> getDimensionNames()
+  public List<String> getDimensionNames(final boolean includeTime)
   {
-    return index.getDimensionNames();
+    return index.getDimensionNames(includeTime);
   }
 
   @Override
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java
index 45d081c8b91..f43b81c40ad 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.segment.incremental;
 
+import org.apache.druid.query.Order;
 import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.extraction.ExtractionFn;
 import org.apache.druid.segment.ColumnSelectorFactory;
@@ -43,20 +44,20 @@ class IncrementalIndexColumnSelectorFactory implements ColumnSelectorFactory, Ro
   private final IncrementalIndexStorageAdapter adapter;
   private final IncrementalIndex index;
   private final VirtualColumns virtualColumns;
-  private final boolean descending;
+  private final Order timeOrder;
   private final IncrementalIndexRowHolder rowHolder;
 
   IncrementalIndexColumnSelectorFactory(
       IncrementalIndexStorageAdapter adapter,
       VirtualColumns virtualColumns,
-      boolean descending,
+      Order timeOrder,
       IncrementalIndexRowHolder rowHolder
   )
   {
     this.adapter = adapter;
     this.index = adapter.index;
     this.virtualColumns = virtualColumns;
-    this.descending = descending;
+    this.timeOrder = timeOrder;
     this.rowHolder = rowHolder;
   }
 
@@ -75,8 +76,12 @@ class IncrementalIndexColumnSelectorFactory implements ColumnSelectorFactory, Ro
     final String dimension = dimensionSpec.getDimension();
     final ExtractionFn extractionFn = dimensionSpec.getExtractionFn();
 
-    if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME)) {
-      return new SingleScanTimeDimensionSelector(makeColumnValueSelector(dimension), extractionFn, descending);
+    if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME) && timeOrder != Order.NONE) {
+      return new SingleScanTimeDimensionSelector(
+          makeColumnValueSelector(dimension),
+          extractionFn,
+          timeOrder
+      );
     }
 
     final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensionSpec.getDimension());
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexCursorHolder.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexCursorHolder.java
index f0966a828eb..33c08d5920c 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexCursorHolder.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexCursorHolder.java
@@ -21,6 +21,7 @@ package org.apache.druid.segment.incremental;
 
 import com.google.common.collect.Iterators;
 import org.apache.druid.query.BaseQuery;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.ValueMatcher;
@@ -34,6 +35,7 @@ import org.apache.druid.segment.filter.ValueMatchers;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -43,7 +45,6 @@ public class IncrementalIndexCursorHolder implements CursorHolder
   private final IncrementalIndex index;
   private final CursorBuildSpec spec;
   private final List<OrderBy> ordering;
-  private final boolean descending;
 
   public IncrementalIndexCursorHolder(
       IncrementalIndexStorageAdapter storageAdapter,
@@ -54,13 +55,16 @@ public class IncrementalIndexCursorHolder implements CursorHolder
     this.storageAdapter = storageAdapter;
     this.index = index;
     this.spec = spec;
-    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
-    if (Cursors.preferDescendingTimeOrdering(spec)) {
-      this.ordering = Cursors.descendingTimeOrder();
-      this.descending = true;
+    if (index.timePosition == 0) {
+      if (Cursors.preferDescendingTimeOrdering(spec)) {
+        this.ordering = Cursors.descendingTimeOrder();
+      } else {
+        this.ordering = Cursors.ascendingTimeOrder();
+      }
     } else {
-      this.ordering = Cursors.ascendingTimeOrder();
-      this.descending = false;
+      // In principle, we could report a sort order here for certain types of fact holders; for example the
+      // RollupFactsHolder would be sorted by dimensions. However, this is left for future work.
+      this.ordering = Collections.emptyList();
     }
   }
 
@@ -80,7 +84,7 @@ public class IncrementalIndexCursorHolder implements CursorHolder
         storageAdapter,
         index,
         spec.getVirtualColumns(),
-        descending,
+        Cursors.getTimeOrdering(ordering),
         spec.getFilter(),
         spec.getInterval()
     );
@@ -109,7 +113,7 @@ public class IncrementalIndexCursorHolder implements CursorHolder
         IncrementalIndexStorageAdapter storageAdapter,
         IncrementalIndex index,
         VirtualColumns virtualColumns,
-        boolean descending,
+        Order timeOrder,
         @Nullable Filter filter,
         Interval actualInterval
     )
@@ -118,7 +122,7 @@ public class IncrementalIndexCursorHolder implements CursorHolder
       columnSelectorFactory = new IncrementalIndexColumnSelectorFactory(
           storageAdapter,
           virtualColumns,
-          descending,
+          timeOrder,
           currEntry
       );
       // Set maxRowIndex before creating the filterMatcher. See https://github.com/apache/druid/pull/6340
@@ -127,7 +131,7 @@ public class IncrementalIndexCursorHolder implements CursorHolder
       numAdvanced = -1;
       facts = index.getFacts();
       cursorIterable = facts.timeRangeIterable(
-          descending,
+          timeOrder == Order.DESCENDING,
           actualInterval.getStartMillis(),
           actualInterval.getEndMillis()
       );
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRow.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRow.java
index 70a589f3950..89e94961f6b 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRow.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRow.java
@@ -142,7 +142,7 @@ public final class IncrementalIndexRow
           @Override
           public Object apply(@Nullable Object input)
           {
-            if (input == null || Array.getLength(input) == 0) {
+            if (input == null || (input.getClass().isArray() && Array.getLength(input) == 0)) {
               return Collections.singletonList("null");
             }
             return Collections.singletonList(input);
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRowIterator.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRowIterator.java
index 98ad5c75109..0e7e82f01c3 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRowIterator.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexRowIterator.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.segment.incremental;
 
+import org.apache.druid.query.Order;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.DimensionHandler;
@@ -67,7 +68,7 @@ class IncrementalIndexRowIterator implements TransformableRowIterator
         new IncrementalIndexColumnSelectorFactory(
             new IncrementalIndexStorageAdapter(incrementalIndex),
             VirtualColumns.EMPTY,
-            false,
+            incrementalIndex.timePosition == 0 ? Order.ASCENDING : Order.NONE,
             rowHolder
         );
     ColumnValueSelector[] dimensionSelectors = incrementalIndex
@@ -91,6 +92,7 @@ class IncrementalIndexRowIterator implements TransformableRowIterator
 
     return new RowPointer(
         rowHolder,
+        incrementalIndex.timePosition,
         dimensionSelectors,
         dimensionHandlers,
         metricSelectors,
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
index ad55e33adae..ab92a00897b 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
@@ -32,12 +32,12 @@ import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.ListIndexed;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
 
 /**
+ *
  */
 public class IncrementalIndexStorageAdapter implements StorageAdapter
 {
@@ -125,7 +125,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
   @Override
   public Indexed<String> getAvailableDimensions()
   {
-    return new ListIndexed<>(index.getDimensionNames());
+    return new ListIndexed<>(index.getDimensionNames(false));
   }
 
   @Override
@@ -155,18 +155,6 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
     return index.size();
   }
 
-  @Override
-  public DateTime getMinTime()
-  {
-    return index.getMinTime();
-  }
-
-  @Override
-  public DateTime getMaxTime()
-  {
-    return index.getMaxTime();
-  }
-
   @Nullable
   @Override
   public Comparable getMinValue(String column)
@@ -236,12 +224,6 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
     );
   }
 
-  @Override
-  public DateTime getMaxIngestedEventTime()
-  {
-    return index.getMaxIngestedEventTime();
-  }
-
   @Override
   public CursorHolder makeCursorHolder(CursorBuildSpec spec)
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/OnheapIncrementalIndex.java
index 0103dc7b729..b5e580f44f2 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/OnheapIncrementalIndex.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/OnheapIncrementalIndex.java
@@ -21,11 +21,13 @@ package org.apache.druid.segment.incremental;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Maps;
 import org.apache.druid.data.input.MapBasedRow;
 import org.apache.druid.data.input.Row;
 import org.apache.druid.error.DruidException;
+import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.java.util.common.logger.Logger;
@@ -46,6 +48,7 @@ import org.apache.druid.utils.JvmUtils;
 
 import javax.annotation.Nullable;
 import java.io.IOException;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
@@ -145,8 +148,13 @@ public class OnheapIncrementalIndex extends IncrementalIndex
     super(incrementalIndexSchema, preserveExistingMetrics, useMaxMemoryEstimates);
     this.maxRowCount = maxRowCount;
     this.maxBytesInMemory = maxBytesInMemory == 0 ? Long.MAX_VALUE : maxBytesInMemory;
-    this.facts = incrementalIndexSchema.isRollup() ? new RollupFactsHolder(dimsComparator(), getDimensions())
-                                                   : new PlainFactsHolder(dimsComparator());
+    if (incrementalIndexSchema.isRollup()) {
+      this.facts = new RollupFactsHolder(dimsComparator(), getDimensions(), timePosition == 0);
+    } else if (timePosition == 0) {
+      this.facts = new PlainTimeOrderedFactsHolder(dimsComparator());
+    } else {
+      this.facts = new PlainNonTimeOrderedFactsHolder(dimsComparator());
+    }
     maxBytesPerRowForAggregators =
         useMaxMemoryEstimates ? getMaxBytesPerRowForAggregators(incrementalIndexSchema) : 0;
     this.useMaxMemoryEstimates = useMaxMemoryEstimates;
@@ -734,14 +742,19 @@ public class OnheapIncrementalIndex extends IncrementalIndex
     // Can't use Set because we need to be able to get from collection
     private final ConcurrentNavigableMap<IncrementalIndexRow, IncrementalIndexRow> facts;
     private final List<DimensionDesc> dimensionDescsList;
+    private final boolean timeOrdered;
+    private volatile long minTime = DateTimes.MAX.getMillis();
+    private volatile long maxTime = DateTimes.MIN.getMillis();
 
     RollupFactsHolder(
         Comparator<IncrementalIndexRow> incrementalIndexRowComparator,
-        List<DimensionDesc> dimensionDescsList
+        List<DimensionDesc> dimensionDescsList,
+        boolean timeOrdered
     )
     {
       this.facts = new ConcurrentSkipListMap<>(incrementalIndexRowComparator);
       this.dimensionDescsList = dimensionDescsList;
+      this.timeOrdered = timeOrdered;
     }
 
     @Override
@@ -754,13 +767,13 @@ public class OnheapIncrementalIndex extends IncrementalIndex
     @Override
     public long getMinTimeMillis()
     {
-      return facts.firstKey().getTimestamp();
+      return minTime;
     }
 
     @Override
     public long getMaxTimeMillis()
     {
-      return facts.lastKey().getTimestamp();
+      return maxTime;
     }
 
     @Override
@@ -777,11 +790,18 @@ public class OnheapIncrementalIndex extends IncrementalIndex
     @Override
     public Iterable<IncrementalIndexRow> timeRangeIterable(boolean descending, long timeStart, long timeEnd)
     {
-      IncrementalIndexRow start = new IncrementalIndexRow(timeStart, new Object[]{}, dimensionDescsList);
-      IncrementalIndexRow end = new IncrementalIndexRow(timeEnd, new Object[]{}, dimensionDescsList);
-      ConcurrentNavigableMap<IncrementalIndexRow, IncrementalIndexRow> subMap = facts.subMap(start, end);
-      ConcurrentMap<IncrementalIndexRow, IncrementalIndexRow> rangeMap = descending ? subMap.descendingMap() : subMap;
-      return rangeMap.keySet();
+      if (timeOrdered) {
+        IncrementalIndexRow start = new IncrementalIndexRow(timeStart, new Object[]{}, dimensionDescsList);
+        IncrementalIndexRow end = new IncrementalIndexRow(timeEnd, new Object[]{}, dimensionDescsList);
+        ConcurrentNavigableMap<IncrementalIndexRow, IncrementalIndexRow> subMap = facts.subMap(start, end);
+        ConcurrentMap<IncrementalIndexRow, IncrementalIndexRow> rangeMap = descending ? subMap.descendingMap() : subMap;
+        return rangeMap.keySet();
+      } else {
+        return Iterables.filter(
+            facts.keySet(),
+            row -> row.timestamp >= timeStart && row.timestamp < timeEnd
+        );
+      }
     }
 
     @Override
@@ -802,6 +822,8 @@ public class OnheapIncrementalIndex extends IncrementalIndex
     {
       // setRowIndex() must be called before facts.putIfAbsent() for visibility of rowIndex from concurrent readers.
       key.setRowIndex(rowIndex);
+      minTime = Math.min(minTime, key.timestamp);
+      maxTime = Math.max(maxTime, key.timestamp);
       IncrementalIndexRow prev = facts.putIfAbsent(key, key);
       return prev == null ? IncrementalIndexRow.EMPTY_ROW_INDEX : prev.getRowIndex();
     }
@@ -813,13 +835,13 @@ public class OnheapIncrementalIndex extends IncrementalIndex
     }
   }
 
-  static final class PlainFactsHolder implements FactsHolder
+  static final class PlainTimeOrderedFactsHolder implements FactsHolder
   {
     private final ConcurrentNavigableMap<Long, Deque<IncrementalIndexRow>> facts;
 
     private final Comparator<IncrementalIndexRow> incrementalIndexRowComparator;
 
-    public PlainFactsHolder(Comparator<IncrementalIndexRow> incrementalIndexRowComparator)
+    public PlainTimeOrderedFactsHolder(Comparator<IncrementalIndexRow> incrementalIndexRowComparator)
     {
       this.facts = new ConcurrentSkipListMap<>();
       this.incrementalIndexRowComparator = incrementalIndexRowComparator;
@@ -918,4 +940,84 @@ public class OnheapIncrementalIndex extends IncrementalIndex
       facts.clear();
     }
   }
+
+  static final class PlainNonTimeOrderedFactsHolder implements FactsHolder
+  {
+    private final Deque<IncrementalIndexRow> facts;
+    private final Comparator<IncrementalIndexRow> incrementalIndexRowComparator;
+    private volatile long minTime = DateTimes.MAX.getMillis();
+    private volatile long maxTime = DateTimes.MIN.getMillis();
+
+    public PlainNonTimeOrderedFactsHolder(Comparator<IncrementalIndexRow> incrementalIndexRowComparator)
+    {
+      this.facts = new ArrayDeque<>();
+      this.incrementalIndexRowComparator = incrementalIndexRowComparator;
+    }
+
+    @Override
+    public int getPriorIndex(IncrementalIndexRow key)
+    {
+      // always return EMPTY_ROW_INDEX to indicate that no prior key cause we always add new row
+      return IncrementalIndexRow.EMPTY_ROW_INDEX;
+    }
+
+    @Override
+    public long getMinTimeMillis()
+    {
+      return minTime;
+    }
+
+    @Override
+    public long getMaxTimeMillis()
+    {
+      return maxTime;
+    }
+
+    @Override
+    public Iterator<IncrementalIndexRow> iterator(boolean descending)
+    {
+      return descending ? facts.descendingIterator() : facts.iterator();
+    }
+
+    @Override
+    public Iterable<IncrementalIndexRow> timeRangeIterable(boolean descending, long timeStart, long timeEnd)
+    {
+      return Iterables.filter(
+          () -> iterator(descending),
+          row -> row.timestamp >= timeStart && row.timestamp < timeEnd
+      );
+    }
+
+    @Override
+    public Iterable<IncrementalIndexRow> keySet()
+    {
+      return facts;
+    }
+
+    @Override
+    public Iterable<IncrementalIndexRow> persistIterable()
+    {
+      final List<IncrementalIndexRow> sortedFacts = new ArrayList<>(facts);
+      sortedFacts.sort(incrementalIndexRowComparator);
+      return sortedFacts;
+    }
+
+    @Override
+    public int putIfAbsent(IncrementalIndexRow key, int rowIndex)
+    {
+      // setRowIndex() must be called before rows.add() for visibility of rowIndex from concurrent readers.
+      key.setRowIndex(rowIndex);
+      minTime = Math.min(minTime, key.timestamp);
+      maxTime = Math.max(maxTime, key.timestamp);
+      facts.add(key);
+      // always return EMPTY_ROW_INDEX to indicate that we always add new row
+      return IncrementalIndexRow.EMPTY_ROW_INDEX;
+    }
+
+    @Override
+    public void clear()
+    {
+      facts.clear();
+    }
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/join/HashJoinEngine.java b/processing/src/main/java/org/apache/druid/segment/join/HashJoinEngine.java
index 6567baa140d..89356a9c9cb 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/HashJoinEngine.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/HashJoinEngine.java
@@ -55,7 +55,6 @@ public class HashJoinEngine
   public static Cursor makeJoinCursor(
       final Cursor leftCursor,
       final JoinableClause joinableClause,
-      final boolean descending,
       final Closer closer
   )
   {
@@ -65,7 +64,6 @@ public class HashJoinEngine
                                                       leftColumnSelectorFactory,
                                                       joinableClause.getCondition(),
                                                       joinableClause.getJoinType().isRighty(),
-                                                      descending,
                                                       closer
                                                   );
 
diff --git a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegment.java b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegment.java
index cdcc3ad8d6b..0cf54b8c148 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegment.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegment.java
@@ -28,6 +28,8 @@ import org.apache.druid.segment.CloseableShapeshifter;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.SegmentReference;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
+import org.apache.druid.segment.WrappedTimeBoundaryInspector;
 import org.apache.druid.segment.join.filter.JoinFilterPreAnalysis;
 import org.apache.druid.timeline.SegmentId;
 import org.apache.druid.utils.CloseableUtils;
@@ -158,7 +160,10 @@ public class HashJoinSegment implements SegmentReference
   {
     if (CloseableShapeshifter.class.equals(clazz)) {
       return (T) new StorageAdapterRowsAndColumns(this.asStorageAdapter());
+    } else if (TimeBoundaryInspector.class.equals(clazz)) {
+      return (T) WrappedTimeBoundaryInspector.create(baseSegment.as(TimeBoundaryInspector.class));
+    } else {
+      return SegmentReference.super.as(clazz);
     }
-    return SegmentReference.super.as(clazz);
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
index 1276a60523f..f3668ca45e1 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
@@ -29,7 +29,6 @@ import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
-import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.Metadata;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
@@ -42,7 +41,6 @@ import org.apache.druid.segment.join.filter.JoinFilterPreAnalysis;
 import org.apache.druid.segment.join.filter.JoinFilterPreAnalysisKey;
 import org.apache.druid.segment.join.filter.JoinFilterSplit;
 import org.apache.druid.utils.CloseableUtils;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -135,18 +133,6 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
     }
   }
 
-  @Override
-  public DateTime getMinTime()
-  {
-    return baseAdapter.getMinTime();
-  }
-
-  @Override
-  public DateTime getMaxTime()
-  {
-    return baseAdapter.getMaxTime();
-  }
-
   @Nullable
   @Override
   public Comparable getMinValue(String column)
@@ -192,12 +178,6 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
     throw new UnsupportedOperationException("Cannot retrieve number of rows from join segment");
   }
 
-  @Override
-  public DateTime getMaxIngestedEventTime()
-  {
-    return baseAdapter.getMaxIngestedEventTime();
-  }
-
   @Override
   public Metadata getMetadata()
   {
@@ -232,23 +212,27 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
       return baseAdapter.makeCursorHolder(newSpec);
     }
 
-    // adequate for time ordering, but needs to be updated if we support cursors ordered other time as the primary
-    final List<OrderBy> ordering;
-    final boolean descending;
-    if (Cursors.preferDescendingTimeOrdering(spec)) {
-      ordering = Cursors.descendingTimeOrder();
-      descending = true;
-    } else {
-      ordering = Cursors.ascendingTimeOrder();
-      descending = false;
-    }
-
     return new CursorHolder()
     {
       final Closer joinablesCloser = Closer.create();
 
-      @Override
-      public Cursor asCursor()
+      /**
+       * Typically the same as {@link HashJoinSegmentStorageAdapter#joinFilterPreAnalysis}, but may differ when
+       * an unnest datasource is layered on top of a join datasource.
+       */
+      final JoinFilterPreAnalysis actualPreAnalysis;
+
+      /**
+       * Result of {@link JoinFilterAnalyzer#splitFilter} on {@link #actualPreAnalysis} and
+       * {@link HashJoinSegmentStorageAdapter#baseFilter}.
+       */
+      final JoinFilterSplit joinFilterSplit;
+
+      /**
+       * Cursor holder for {@link HashJoinSegmentStorageAdapter#baseAdapter}.
+       */
+      final CursorHolder baseCursorHolder;
+
       {
         // Filter pre-analysis key implied by the call to "makeCursorHolder". We need to sanity-check that it matches
         // the actual pre-analysis that was done. Note: we could now infer a rewrite config from the "makeCursorHolder"
@@ -264,23 +248,21 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
             );
 
         final JoinFilterPreAnalysisKey keyCached = joinFilterPreAnalysis.getKey();
-        final JoinFilterPreAnalysis preAnalysis;
         if (keyIn.equals(keyCached)) {
           // Common case: key used during filter pre-analysis (keyCached) matches key implied by makeCursorHolder call
           // (keyIn).
-          preAnalysis = joinFilterPreAnalysis;
+          actualPreAnalysis = joinFilterPreAnalysis;
         } else {
           // Less common case: key differs. Re-analyze the filter. This case can happen when an unnest datasource is
           // layered on top of a join datasource.
-          preAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(keyIn);
+          actualPreAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(keyIn);
         }
 
-        final JoinFilterSplit joinFilterSplit = JoinFilterAnalyzer.splitFilter(
-            preAnalysis,
+        joinFilterSplit = JoinFilterAnalyzer.splitFilter(
+            actualPreAnalysis,
             baseFilter
         );
 
-
         if (joinFilterSplit.getBaseTableFilter().isPresent()) {
           cursorBuildSpecBuilder.setFilter(joinFilterSplit.getBaseTableFilter().get());
         }
@@ -295,8 +277,14 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
         );
         cursorBuildSpecBuilder.setVirtualColumns(preJoinVirtualColumns);
 
-        final Cursor baseCursor = joinablesCloser.register(baseAdapter.makeCursorHolder(cursorBuildSpecBuilder.build()))
-                                                 .asCursor();
+        baseCursorHolder =
+            joinablesCloser.register(baseAdapter.makeCursorHolder(cursorBuildSpecBuilder.build()));
+      }
+
+      @Override
+      public Cursor asCursor()
+      {
+        final Cursor baseCursor = baseCursorHolder.asCursor();
 
         if (baseCursor == null) {
           return null;
@@ -305,21 +293,20 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
         Cursor retVal = baseCursor;
 
         for (JoinableClause clause : clauses) {
-          retVal = HashJoinEngine.makeJoinCursor(retVal, clause, descending, joinablesCloser);
+          retVal = HashJoinEngine.makeJoinCursor(retVal, clause, joinablesCloser);
         }
 
         return PostJoinCursor.wrap(
             retVal,
-            VirtualColumns.fromIterable(preAnalysis.getPostJoinVirtualColumns()),
+            VirtualColumns.fromIterable(actualPreAnalysis.getPostJoinVirtualColumns()),
             joinFilterSplit.getJoinTableFilter().orElse(null)
         );
       }
 
-      @Nullable
       @Override
       public List<OrderBy> getOrdering()
       {
-        return ordering;
+        return computeOrdering(baseCursorHolder.getOrdering());
       }
 
       @Override
@@ -367,4 +354,20 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
   {
     return Filters.maybeAnd(Arrays.asList(baseFilter, other)).orElse(null);
   }
+
+  /**
+   * Computes ordering of a join {@link CursorHolder} based on the ordering of an underlying {@link CursorHolder}.
+   */
+  private List<OrderBy> computeOrdering(final List<OrderBy> baseOrdering)
+  {
+    // Sorted the same way as the base segment, unless a joined-in column shadows one of the base columns.
+    int limit = 0;
+    for (; limit < baseOrdering.size(); limit++) {
+      if (!isBaseColumn(baseOrdering.get(limit).getColumnName())) {
+        break;
+      }
+    }
+
+    return limit == baseOrdering.size() ? baseOrdering : baseOrdering.subList(0, limit);
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/join/Joinable.java b/processing/src/main/java/org/apache/druid/segment/join/Joinable.java
index dab102d4493..6886065f1a6 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/Joinable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/Joinable.java
@@ -72,7 +72,6 @@ public interface Joinable extends ReferenceCountedObject
    * @param condition                 join condition for the matcher
    * @param remainderNeeded           whether or not {@link JoinMatcher#matchRemainder()} will ever be called on the
    *                                  matcher. If we know it will not, additional optimizations are often possible.
-   * @param descending                true if join cursor is iterated in descending order
    * @param closer                    closer that will run after join cursor has completed to clean up any per query
    *                                  resources the joinable uses
    *
@@ -82,7 +81,6 @@ public interface Joinable extends ReferenceCountedObject
       ColumnSelectorFactory leftColumnSelectorFactory,
       JoinConditionAnalysis condition,
       boolean remainderNeeded,
-      boolean descending,
       Closer closer
   );
 
diff --git a/processing/src/main/java/org/apache/druid/segment/join/lookup/LookupJoinable.java b/processing/src/main/java/org/apache/druid/segment/join/lookup/LookupJoinable.java
index 638506e5901..f0c9a34526d 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/lookup/LookupJoinable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/lookup/LookupJoinable.java
@@ -90,7 +90,6 @@ public class LookupJoinable implements Joinable
       final ColumnSelectorFactory leftSelectorFactory,
       final JoinConditionAnalysis condition,
       final boolean remainderNeeded,
-      boolean descending,
       Closer closer
   )
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java b/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
index 716decb04d9..9591dc315f0 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
@@ -30,6 +30,7 @@ import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.NilColumnValueSelector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexColumnSelectorFactory;
@@ -222,11 +223,11 @@ public class BroadcastSegmentIndexedTable implements IndexedTable
 
   @Nullable
   @Override
-  public ColumnSelectorFactory makeColumnSelectorFactory(ReadableOffset offset, boolean descending, Closer closer)
+  public ColumnSelectorFactory makeColumnSelectorFactory(ReadableOffset offset, Closer closer)
   {
     return new QueryableIndexColumnSelectorFactory(
         VirtualColumns.EMPTY,
-        descending,
+        Cursors.getTimeOrdering(queryableIndex.getOrdering()),
         offset,
         new ColumnCache(queryableIndex, closer)
     );
diff --git a/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTable.java b/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTable.java
index 221fa67fc0f..e2c01cd7fb2 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTable.java
@@ -85,7 +85,7 @@ public interface IndexedTable extends ReferenceCountedObject, Closeable
    * instead.
    */
   @Nullable
-  default ColumnSelectorFactory makeColumnSelectorFactory(ReadableOffset offset, boolean descending, Closer closer)
+  default ColumnSelectorFactory makeColumnSelectorFactory(ReadableOffset offset, Closer closer)
   {
     return null;
   }
diff --git a/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTableJoinMatcher.java b/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTableJoinMatcher.java
index f96e4260f8e..3f3c75770f0 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTableJoinMatcher.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTableJoinMatcher.java
@@ -46,7 +46,6 @@ import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.SimpleAscendingOffset;
-import org.apache.druid.segment.SimpleDescendingOffset;
 import org.apache.druid.segment.SimpleSettableOffset;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnType;
@@ -95,16 +94,11 @@ public class IndexedTableJoinMatcher implements JoinMatcher
       final ColumnSelectorFactory leftSelectorFactory,
       final JoinConditionAnalysis condition,
       final boolean remainderNeeded,
-      final boolean descending,
       final Closer closer
   )
   {
     this.table = table;
-    if (descending) {
-      this.joinableOffset = new SimpleDescendingOffset(table.numRows());
-    } else {
-      this.joinableOffset = new SimpleAscendingOffset(table.numRows());
-    }
+    this.joinableOffset = new SimpleAscendingOffset(table.numRows());
     reset();
 
     if (condition.isAlwaysTrue()) {
@@ -133,7 +127,7 @@ public class IndexedTableJoinMatcher implements JoinMatcher
       );
     }
 
-    ColumnSelectorFactory selectorFactory = table.makeColumnSelectorFactory(joinableOffset, descending, closer);
+    ColumnSelectorFactory selectorFactory = table.makeColumnSelectorFactory(joinableOffset, closer);
     this.selectorFactory = selectorFactory != null
                            ? selectorFactory
                            : new IndexedTableColumnSelectorFactory(table, () -> currentRow, closer);
diff --git a/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTableJoinable.java b/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTableJoinable.java
index 4e9c5b5b352..10384b373a6 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTableJoinable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/table/IndexedTableJoinable.java
@@ -77,7 +77,6 @@ public class IndexedTableJoinable implements Joinable
       final ColumnSelectorFactory leftColumnSelectorFactory,
       final JoinConditionAnalysis condition,
       final boolean remainderNeeded,
-      boolean descending,
       Closer closer
   )
   {
@@ -86,7 +85,6 @@ public class IndexedTableJoinable implements Joinable
         leftColumnSelectorFactory,
         condition,
         remainderNeeded,
-        descending,
         closer
     );
   }
diff --git a/processing/src/main/java/org/apache/druid/segment/loading/TombstoneSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/TombstoneSegmentizerFactory.java
index 72958e8049d..bb6ad8aaca9 100644
--- a/processing/src/main/java/org/apache/druid/segment/loading/TombstoneSegmentizerFactory.java
+++ b/processing/src/main/java/org/apache/druid/segment/loading/TombstoneSegmentizerFactory.java
@@ -22,6 +22,7 @@ package org.apache.druid.segment.loading;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.druid.collections.bitmap.BitmapFactory;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.DimensionHandler;
 import org.apache.druid.segment.Metadata;
 import org.apache.druid.segment.QueryableIndex;
@@ -36,7 +37,6 @@ import org.apache.druid.timeline.SegmentId;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
-
 import java.io.File;
 import java.util.List;
 import java.util.Map;
@@ -102,6 +102,12 @@ public class TombstoneSegmentizerFactory implements SegmentizerFactory
             throw new UnsupportedOperationException();
           }
 
+          @Override
+          public List<OrderBy> getOrdering()
+          {
+            throw new UnsupportedOperationException();
+          }
+
           @Override
           public void close()
           {
diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
index 979d213ee41..5aac2a2a9e8 100644
--- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
+++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
@@ -205,10 +205,14 @@ public class ExpressionSelectors
       final String column = plan.getSingleInputName();
       final ColumnType inputType = plan.getSingleInputType();
       if (inputType.is(ValueType.LONG)) {
+        // Skip LRU cache when the underlying data is sorted by __time. Note: data is not always sorted by __time; when
+        // forceSegmentSortByTime: false, segments can be written in non-__time order. However, this
+        // information is not currently available here, so we assume the common case, which is __time-sortedness.
+        final boolean useLruCache = !ColumnHolder.TIME_COLUMN_NAME.equals(column);
         return new SingleLongInputCachingExpressionColumnValueSelector(
             columnSelectorFactory.makeColumnValueSelector(column),
             plan.getExpression(),
-            !ColumnHolder.TIME_COLUMN_NAME.equals(column), // __time doesn't need an LRU cache since it is sorted.
+            useLruCache,
             rowIdSupplier
         );
       } else if (inputType.is(ValueType.STRING)) {
diff --git a/processing/src/test/java/org/apache/druid/frame/segment/FrameStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/frame/segment/FrameStorageAdapterTest.java
index b9abf2f81b1..2a8f525bf78 100644
--- a/processing/src/test/java/org/apache/druid/frame/segment/FrameStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/segment/FrameStorageAdapterTest.java
@@ -53,9 +53,11 @@ import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.segment.vector.VectorCursor;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
 import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.hamcrest.CoreMatchers;
 import org.joda.time.Interval;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -205,18 +207,6 @@ public class FrameStorageAdapterTest
       Assert.assertNull(frameAdapter.getColumnCapabilities("nonexistent"));
     }
 
-    @Test
-    public void test_getMinTime()
-    {
-      Assert.assertEquals(queryableAdapter.getInterval().getStart(), frameAdapter.getMinTime());
-    }
-
-    @Test
-    public void test_getMaxTime()
-    {
-      Assert.assertEquals(queryableAdapter.getInterval().getEnd().minus(1), frameAdapter.getMaxTime());
-    }
-
     @Test
     public void test_getNumRows()
     {
@@ -267,13 +257,14 @@ public class FrameStorageAdapterTest
       this.interval = interval;
       this.virtualColumns = virtualColumns;
       this.descending = descending;
-      this.buildSpec = CursorBuildSpec.builder()
-                                      .setFilter(this.filter)
-                                      .setInterval(this.interval)
-                                      .setVirtualColumns(this.virtualColumns)
-                                      .setPreferredOrdering(descending ? Cursors.descendingTimeOrder() : null)
-                                      .setQueryContext(queryContext)
-                                      .build();
+      this.buildSpec =
+          CursorBuildSpec.builder()
+                         .setFilter(this.filter)
+                         .setInterval(this.interval)
+                         .setVirtualColumns(this.virtualColumns)
+                         .setPreferredOrdering(descending ? Cursors.descendingTimeOrder() : Collections.emptyList())
+                         .setQueryContext(queryContext)
+                         .build();
     }
 
     @Parameterized.Parameters(name = "frameType = {0}, "
@@ -363,44 +354,49 @@ public class FrameStorageAdapterTest
     @Test
     public void test_makeCursor()
     {
-      assertCursorMatch(adapter -> adapter.makeCursorHolder(buildSpec));
+      final RowSignature signature = frameAdapter.getRowSignature();
+
+      // Frame adapters don't know the order of the underlying frames, so they should ignore the "preferred ordering"
+      // of the cursor build spec. We test this by passing the frameAdapter a build spec with a preferred ordering,
+      // and passing the queryableAdapter the same build spec *without* a preferred ordering, and verifying they match.
+      final CursorBuildSpec queryableBuildSpec =
+          CursorBuildSpec.builder(buildSpec).setPreferredOrdering(Collections.emptyList()).build();
+
+      try (final CursorHolder queryableCursorHolder = queryableAdapter.makeCursorHolder(queryableBuildSpec);
+           final CursorHolder frameCursorHolder = frameAdapter.makeCursorHolder(buildSpec)) {
+        final Sequence<List<Object>> queryableRows =
+            FrameTestUtil.readRowsFromCursor(advanceAndReset(queryableCursorHolder.asCursor()), signature);
+        final Sequence<List<Object>> frameRows =
+            FrameTestUtil.readRowsFromCursor(advanceAndReset(frameCursorHolder.asCursor()), signature);
+        FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
+      }
     }
 
     @Test
     public void test_makeVectorCursor()
     {
+      // Conditions for frames to be vectorizable.
+      Assume.assumeThat(frameType, CoreMatchers.equalTo(FrameType.COLUMNAR));
+      Assume.assumeFalse(descending);
       assertVectorCursorsMatch(adapter -> adapter.makeCursorHolder(buildSpec));
     }
 
-    private void assertCursorMatch(final Function<StorageAdapter, CursorHolder> call)
-    {
-      final RowSignature signature = frameAdapter.getRowSignature();
-      try (final CursorHolder queryableMaker = call.apply(queryableAdapter);
-           final CursorHolder frameMaker = call.apply(frameAdapter)) {
-        final Sequence<List<Object>> queryableRows =
-            FrameTestUtil.readRowsFromCursor(advanceAndReset(queryableMaker.asCursor()), signature);
-        final Sequence<List<Object>> frameRows =
-            FrameTestUtil.readRowsFromCursor(advanceAndReset(frameMaker.asCursor()), signature);
-        FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
-      }
-    }
-
     private void assertVectorCursorsMatch(final Function<StorageAdapter, CursorHolder> call)
     {
       final CursorHolder cursorHolder = call.apply(queryableAdapter);
       final CursorHolder frameCursorHolder = call.apply(frameAdapter);
-      if (frameCursorHolder.canVectorize()) {
-        final RowSignature signature = frameAdapter.getRowSignature();
-        final Sequence<List<Object>> queryableRows =
-            FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(cursorHolder.asVectorCursor()), signature).withBaggage(cursorHolder);
-        final Sequence<List<Object>> frameRows =
-            FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(frameCursorHolder.asVectorCursor()), signature)
-                         .withBaggage(frameCursorHolder);
-        FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
-      } else {
-        cursorHolder.close();
-        frameCursorHolder.close();
-      }
+
+      Assert.assertTrue("queryable cursor is vectorizable", cursorHolder.canVectorize());
+      Assert.assertTrue("frame cursor is vectorizable", frameCursorHolder.canVectorize());
+
+      final RowSignature signature = frameAdapter.getRowSignature();
+      final Sequence<List<Object>> queryableRows =
+          FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(cursorHolder.asVectorCursor()), signature)
+                       .withBaggage(cursorHolder);
+      final Sequence<List<Object>> frameRows =
+          FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(frameCursorHolder.asVectorCursor()), signature)
+                       .withBaggage(frameCursorHolder);
+      FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
     }
 
     /**
diff --git a/processing/src/test/java/org/apache/druid/query/CursorGranularizerTest.java b/processing/src/test/java/org/apache/druid/query/CursorGranularizerTest.java
index 8aeaf626070..75119bfb897 100644
--- a/processing/src/test/java/org/apache/druid/query/CursorGranularizerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/CursorGranularizerTest.java
@@ -33,8 +33,11 @@ import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.IndexBuilder;
+import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexTimeBoundaryInspector;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.incremental.IncrementalIndexSchema;
@@ -55,6 +58,7 @@ public class CursorGranularizerTest extends InitializedNullHandlingTest
   public final TemporaryFolder temporaryFolder = new TemporaryFolder();
 
   private StorageAdapter adapter;
+  private TimeBoundaryInspector timeBoundaryInspector;
 
   @Before
   public void setup() throws IOException
@@ -147,7 +151,9 @@ public class CursorGranularizerTest extends InitializedNullHandlingTest
                     )
                     .tmpDir(temporaryFolder.newFolder());
 
-    adapter = new QueryableIndexStorageAdapter(bob.buildMMappedIndex());
+    final QueryableIndex index = bob.buildMMappedIndex();
+    adapter = new QueryableIndexStorageAdapter(index);
+    timeBoundaryInspector = QueryableIndexTimeBoundaryInspector.create(index);
   }
 
   @Test
@@ -156,11 +162,11 @@ public class CursorGranularizerTest extends InitializedNullHandlingTest
     try (CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
       final Cursor cursor = cursorHolder.asCursor();
       CursorGranularizer granularizer = CursorGranularizer.create(
-          adapter,
           cursor,
+          timeBoundaryInspector,
+          Order.ASCENDING,
           Granularities.HOUR,
-          adapter.getInterval(),
-          false
+          adapter.getInterval()
       );
 
       final ColumnSelectorFactory selectorFactory = cursor.getColumnSelectorFactory();
@@ -209,11 +215,11 @@ public class CursorGranularizerTest extends InitializedNullHandlingTest
     try (CursorHolder cursorHolder = adapter.makeCursorHolder(descending)) {
       final Cursor cursor = cursorHolder.asCursor();
       CursorGranularizer granularizer = CursorGranularizer.create(
-          adapter,
           cursor,
+          timeBoundaryInspector,
+          Order.DESCENDING,
           Granularities.HOUR,
-          adapter.getInterval(),
-          true
+          adapter.getInterval()
       );
 
       final ColumnSelectorFactory selectorFactory = cursor.getColumnSelectorFactory();
diff --git a/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java
index e562f72955e..270a335bc77 100644
--- a/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java
+++ b/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java
@@ -393,78 +393,128 @@ public class QueryRunnerTestHelper
   }
 
   /**
-   * Check if a QueryRunner returned by {@link #makeQueryRunners(QueryRunnerFactory)} is vectorizable.
+   * Check if a QueryRunner returned by {@link #makeQueryRunners(QueryRunnerFactory, boolean)} is vectorizable.
    */
   public static boolean isTestRunnerVectorizable(QueryRunner runner)
   {
     final String runnerName = runner.toString();
-    return !("rtIndex".equals(runnerName) || "noRollupRtIndex".equals(runnerName));
+    return !("rtIndex".equals(runnerName)
+             || "noRollupRtIndex".equals(runnerName)
+             || "nonTimeOrderedRtIndex".equals(runnerName)
+             || "nonTimeOrderedNoRollupRtIndex".equals(runnerName));
   }
 
-
-  public static <T, QueryType extends Query<T>> List<QueryRunner<T>> makeQueryRunners(
-      QueryRunnerFactory<T, QueryType> factory
+  /**
+   * Create test query runners.
+   *
+   * @param factory query runner factory
+   * @param includeNonTimeOrdered whether to include runners with non-time-ordered segments. Some test suites are not
+   *                              compatible with non-time-ordered segments.
+   */
+  public static <T, QueryType extends Query<T>> List<TestQueryRunner<T>> makeQueryRunners(
+      QueryRunnerFactory<T, QueryType> factory,
+      boolean includeNonTimeOrdered
   )
   {
-    BiFunction<String, Segment, QueryRunner<T>> maker = (name, seg) -> makeQueryRunner(factory, seg, name);
+    BiFunction<String, Segment, TestQueryRunner<T>> maker = (name, seg) -> makeQueryRunner(factory, seg, name);
 
-    return ImmutableList.of(
-        maker.apply(
-            "rtIndex",
-            new IncrementalIndexSegment(TestIndex.getIncrementalTestIndex(), SEGMENT_ID)
-        ),
-        maker.apply(
-            "noRollupRtIndex",
-            new IncrementalIndexSegment(TestIndex.getNoRollupIncrementalTestIndex(), SEGMENT_ID)
-        ),
-        maker.apply(
-            "mMappedTestIndex",
-            new QueryableIndexSegment(TestIndex.getMMappedTestIndex(), SEGMENT_ID)
-        ),
-        maker.apply(
-            "noRollupMMappedTestIndex",
-            new QueryableIndexSegment(TestIndex.getNoRollupMMappedTestIndex(), SEGMENT_ID)
-        ),
-        maker.apply(
-            "mergedRealtimeIndex",
-            new QueryableIndexSegment(TestIndex.mergedRealtimeIndex(), SEGMENT_ID)
-        ),
-        maker.apply(
-            "frontCodedMMappedTestIndex",
-            new QueryableIndexSegment(TestIndex.getFrontCodedMMappedTestIndex(), SEGMENT_ID)
+    final ImmutableList.Builder<TestQueryRunner<T>> retVal = ImmutableList.builder();
+
+    retVal.addAll(
+        Arrays.asList(
+            maker.apply(
+                "rtIndex",
+                new IncrementalIndexSegment(TestIndex.getIncrementalTestIndex(), SEGMENT_ID)
+            ),
+            maker.apply(
+                "noRollupRtIndex",
+                new IncrementalIndexSegment(TestIndex.getNoRollupIncrementalTestIndex(), SEGMENT_ID)
+            ),
+            maker.apply(
+                "mMappedTestIndex",
+                new QueryableIndexSegment(TestIndex.getMMappedTestIndex(), SEGMENT_ID)
+            ),
+            maker.apply(
+                "noRollupMMappedTestIndex",
+                new QueryableIndexSegment(TestIndex.getNoRollupMMappedTestIndex(), SEGMENT_ID)
+            ),
+            maker.apply(
+                "mergedRealtimeIndex",
+                new QueryableIndexSegment(TestIndex.mergedRealtimeIndex(), SEGMENT_ID)
+            ),
+            maker.apply(
+                "frontCodedMMappedTestIndex",
+                new QueryableIndexSegment(TestIndex.getFrontCodedMMappedTestIndex(), SEGMENT_ID)
+            )
         )
     );
+
+    if (includeNonTimeOrdered) {
+      retVal.addAll(
+          Arrays.asList(
+              maker.apply(
+                  "nonTimeOrderedRtIndex",
+                  new IncrementalIndexSegment(TestIndex.getNonTimeOrderedRealtimeTestIndex(), SEGMENT_ID)
+              ),
+              maker.apply(
+                  "nonTimeOrderedNoRollupRtIndex",
+                  new IncrementalIndexSegment(TestIndex.getNonTimeOrderedNoRollupRealtimeTestIndex(), SEGMENT_ID)
+              ),
+              maker.apply(
+                  "nonTimeOrderedMMappedTestIndex",
+                  new QueryableIndexSegment(TestIndex.getNonTimeOrderedMMappedTestIndex(), SEGMENT_ID)
+              ),
+              maker.apply(
+                  "nonTimeOrderedNoRollupMMappedTestIndex",
+                  new QueryableIndexSegment(TestIndex.getNonTimeOrderedNoRollupMMappedTestIndex(), SEGMENT_ID)
+              )
+          )
+      );
+    }
+
+    return retVal.build();
   }
 
-  public static <T, QueryType extends Query<T>> List<QueryRunner<T>> makeQueryRunnersToMerge(
-      QueryRunnerFactory<T, QueryType> factory
+  /**
+   * Create test query runners.
+   *
+   * @param factory query runner factory
+   * @param includeNonTimeOrdered whether to include runners with non-time-ordered segments. Some test suites are not
+   *                              written to be compatible with non-time-ordered segments.
+   */
+  public static <T, QueryType extends Query<T>> List<TestQueryRunner<T>> makeQueryRunnersToMerge(
+      final QueryRunnerFactory<T, QueryType> factory,
+      final boolean includeNonTimeOrdered
   )
   {
-    return mapQueryRunnersToMerge(factory, makeQueryRunners(factory));
+    return mapQueryRunnersToMerge(factory, makeQueryRunners(factory, includeNonTimeOrdered));
   }
 
-  public static <T, QueryType extends Query<T>> ArrayList<QueryRunner<T>> mapQueryRunnersToMerge(
+  public static <T, QueryType extends Query<T>> ArrayList<TestQueryRunner<T>> mapQueryRunnersToMerge(
       QueryRunnerFactory<T, QueryType> factory,
-      List<QueryRunner<T>> runners
+      List<TestQueryRunner<T>> runners
   )
   {
-    final ArrayList<QueryRunner<T>> retVal = new ArrayList<>(runners.size());
+    final ArrayList<TestQueryRunner<T>> retVal = new ArrayList<>(runners.size());
 
     final QueryToolChest<T, QueryType> toolchest = factory.getToolchest();
-    for (QueryRunner<T> baseRunner : runners) {
+    for (TestQueryRunner<T> baseRunner : runners) {
       retVal.add(
-          FluentQueryRunner.create(baseRunner, toolchest)
-                           .applyPreMergeDecoration()
-                           .mergeResults(true)
-                           .applyPostMergeDecoration()
-                           .setToString(baseRunner.toString())
+          new TestQueryRunner<>(
+              baseRunner.getName(),
+              FluentQueryRunner.create(baseRunner, toolchest)
+                               .applyPreMergeDecoration()
+                               .mergeResults(true)
+                               .applyPostMergeDecoration(),
+              baseRunner.getSegment()
+          )
       );
     }
 
     return retVal;
   }
 
-  public static <T, QueryType extends Query<T>> QueryRunner<T> makeQueryRunner(
+  public static <T, QueryType extends Query<T>> TestQueryRunner<T> makeQueryRunner(
       QueryRunnerFactory<T, QueryType> factory,
       String resourceFileName,
       final String runnerName
@@ -478,7 +528,7 @@ public class QueryRunnerTestHelper
     );
   }
 
-  public static <T, QueryType extends Query<T>> QueryRunner<T> makeQueryRunner(
+  public static <T, QueryType extends Query<T>> TestQueryRunner<T> makeQueryRunner(
       QueryRunnerFactory<T, QueryType> factory,
       Segment adapter,
       final String runnerName
@@ -487,33 +537,24 @@ public class QueryRunnerTestHelper
     return makeQueryRunner(factory, SEGMENT_ID, adapter, runnerName);
   }
 
-  public static <T, QueryType extends Query<T>> QueryRunner<T> makeQueryRunner(
-      QueryRunnerFactory<T, QueryType> factory,
-      SegmentId segmentId,
-      Segment adapter,
+  public static <T, QueryType extends Query<T>> TestQueryRunner<T> makeQueryRunner(
+      final QueryRunnerFactory<T, QueryType> factory,
+      final SegmentId segmentId,
+      final Segment adapter,
       final String runnerName
   )
   {
-    //noinspection
-    return new BySegmentQueryRunner<T>(
-        segmentId,
-        adapter.getDataInterval().getStart(),
-        factory.createRunner(adapter)
-    )
-    {
-      @Override
-      public String toString()
-      {
-        // Tests that use these QueryRunners directly are parameterized and use the toString of their QueryRunner as
-        // the name of the test.  It would be better if the usages were adjusted to actually parameterize with an extra
-        // name parameter, or use a different object or something like that, but for now, we have to overload toString
-        // to name it so that the parameterization continues to work.
-        return runnerName;
-      }
-    };
+    return new TestQueryRunner<>(
+        runnerName,
+        new BySegmentQueryRunner<>(
+            segmentId,
+            adapter.getDataInterval().getStart(),
+            factory.createRunner(adapter)
+        ),
+        adapter
+    );
   }
 
-
   public static <T, QueryType extends Query<T>> QueryRunner<T> makeQueryRunnerWithSegmentMapFn(
       QueryRunnerFactory<T, QueryType> factory,
       Segment adapter,
diff --git a/processing/src/test/java/org/apache/druid/query/TestQueryRunner.java b/processing/src/test/java/org/apache/druid/query/TestQueryRunner.java
new file mode 100644
index 00000000000..1a589d50701
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/query/TestQueryRunner.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query;
+
+import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.query.context.ResponseContext;
+import org.apache.druid.segment.Segment;
+
+/**
+ * Wrapper around a {@link QueryRunner} that adds a name and allows retrieval of the underlying segment. Used by
+ * {@link QueryRunnerTestHelper#makeQueryRunners(QueryRunnerFactory, boolean)} and related methods.
+ */
+public class TestQueryRunner<T> implements QueryRunner<T>
+{
+  private final String name;
+  private final QueryRunner<T> runner;
+  private final Segment segment;
+
+  public TestQueryRunner(String name, QueryRunner<T> runner, Segment segment)
+  {
+    this.name = name;
+    this.runner = runner;
+    this.segment = segment;
+  }
+
+  @Override
+  public Sequence<T> run(QueryPlus<T> queryPlus, ResponseContext responseContext)
+  {
+    return runner.run(queryPlus, responseContext);
+  }
+
+  @Override
+  public Sequence<T> run(QueryPlus<T> queryPlus)
+  {
+    return runner.run(queryPlus);
+  }
+
+  public String getName()
+  {
+    return name;
+  }
+
+  public Segment getSegment()
+  {
+    return segment;
+  }
+
+  @Override
+  public String toString()
+  {
+    return name;
+  }
+}
diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstTimeseriesQueryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstTimeseriesQueryTest.java
index 2e960e6186e..2c44d080deb 100644
--- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstTimeseriesQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstTimeseriesQueryTest.java
@@ -35,8 +35,10 @@ import org.apache.druid.query.timeseries.DefaultTimeseriesQueryMetrics;
 import org.apache.druid.query.timeseries.TimeseriesQuery;
 import org.apache.druid.query.timeseries.TimeseriesQueryEngine;
 import org.apache.druid.query.timeseries.TimeseriesResultValue;
+import org.apache.druid.segment.IncrementalIndexTimeBoundaryInspector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexTimeBoundaryInspector;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.TestIndex;
 import org.apache.druid.segment.incremental.IncrementalIndex;
@@ -143,10 +145,20 @@ public class StringFirstTimeseriesQueryTest extends InitializedNullHandlingTest
 
     final DefaultTimeseriesQueryMetrics defaultTimeseriesQueryMetrics = new DefaultTimeseriesQueryMetrics();
     final Iterable<Result<TimeseriesResultValue>> iiResults =
-        engine.process(query, new IncrementalIndexStorageAdapter(incrementalIndex), defaultTimeseriesQueryMetrics).toList();
+        engine.process(
+            query,
+            new IncrementalIndexStorageAdapter(incrementalIndex),
+            new IncrementalIndexTimeBoundaryInspector(incrementalIndex),
+            defaultTimeseriesQueryMetrics
+        ).toList();
 
     final Iterable<Result<TimeseriesResultValue>> qiResults =
-        engine.process(query, new QueryableIndexStorageAdapter(queryableIndex), defaultTimeseriesQueryMetrics).toList();
+        engine.process(
+            query,
+            new QueryableIndexStorageAdapter(queryableIndex),
+            QueryableIndexTimeBoundaryInspector.create(queryableIndex),
+            defaultTimeseriesQueryMetrics
+        ).toList();
 
     TestHelper.assertExpectedResults(expectedResults, iiResults, "incremental index");
     TestHelper.assertExpectedResults(expectedResults, qiResults, "queryable index");
diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastTimeseriesQueryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastTimeseriesQueryTest.java
index 6bbada405b9..1c9b4033cd9 100644
--- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastTimeseriesQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastTimeseriesQueryTest.java
@@ -35,8 +35,10 @@ import org.apache.druid.query.timeseries.DefaultTimeseriesQueryMetrics;
 import org.apache.druid.query.timeseries.TimeseriesQuery;
 import org.apache.druid.query.timeseries.TimeseriesQueryEngine;
 import org.apache.druid.query.timeseries.TimeseriesResultValue;
+import org.apache.druid.segment.IncrementalIndexTimeBoundaryInspector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexTimeBoundaryInspector;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.TestIndex;
 import org.apache.druid.segment.incremental.IncrementalIndex;
@@ -147,10 +149,20 @@ public class StringLastTimeseriesQueryTest extends InitializedNullHandlingTest
 
     final DefaultTimeseriesQueryMetrics defaultTimeseriesQueryMetrics = new DefaultTimeseriesQueryMetrics();
     final Iterable<Result<TimeseriesResultValue>> iiResults =
-        engine.process(query, new IncrementalIndexStorageAdapter(incrementalIndex), defaultTimeseriesQueryMetrics).toList();
+        engine.process(
+            query,
+            new IncrementalIndexStorageAdapter(incrementalIndex),
+            new IncrementalIndexTimeBoundaryInspector(incrementalIndex),
+            defaultTimeseriesQueryMetrics
+        ).toList();
 
     final Iterable<Result<TimeseriesResultValue>> qiResults =
-        engine.process(query, new QueryableIndexStorageAdapter(queryableIndex), defaultTimeseriesQueryMetrics).toList();
+        engine.process(
+            query,
+            new QueryableIndexStorageAdapter(queryableIndex),
+            QueryableIndexTimeBoundaryInspector.create(queryableIndex),
+            defaultTimeseriesQueryMetrics
+        ).toList();
 
     TestHelper.assertExpectedResults(expectedResults, iiResults, "incremental index");
     TestHelper.assertExpectedResults(expectedResults, qiResults, "queryable index");
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java
index 7c5b5e90844..5de602b8772 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java
@@ -173,7 +173,7 @@ public class GroupByQueryMergeBufferTest extends InitializedNullHandlingTest
   public static Collection<Object[]> constructorFeeder()
   {
     final List<Object[]> args = new ArrayList<>();
-    for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(FACTORY)) {
+    for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(FACTORY, true)) {
       args.add(new Object[]{runner});
     }
     return args;
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerFailureTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerFailureTest.java
index ad6157dc927..69debeb902c 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerFailureTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerFailureTest.java
@@ -40,7 +40,10 @@ import org.apache.druid.query.ResourceLimitExceededException;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.segment.TestHelper;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -126,6 +129,26 @@ public class GroupByQueryRunnerFailureTest
 
   private QueryRunner<ResultRow> runner;
 
+  @Before
+  public void setUp()
+  {
+    Assert.assertEquals(
+        "MERGE_BUFFER_POOL size, pre-test",
+        MERGE_BUFFER_POOL.maxSize(),
+        MERGE_BUFFER_POOL.getPoolSize()
+    );
+  }
+
+  @After
+  public void tearDown()
+  {
+    Assert.assertEquals(
+        "MERGE_BUFFER_POOL size, post-test",
+        MERGE_BUFFER_POOL.maxSize(),
+        MERGE_BUFFER_POOL.getPoolSize()
+    );
+  }
+
   @AfterClass
   public static void teardownClass()
   {
@@ -137,7 +160,7 @@ public class GroupByQueryRunnerFailureTest
   public static Collection<Object[]> constructorFeeder()
   {
     final List<Object[]> args = new ArrayList<>();
-    for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(FACTORY)) {
+    for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(FACTORY, true)) {
       args.add(new Object[]{runner});
     }
     return args;
@@ -253,15 +276,13 @@ public class GroupByQueryRunnerFailureTest
   @Test(timeout = 60_000L)
   public void testTimeoutExceptionOnQueryable()
   {
-    expectedException.expect(QueryTimeoutException.class);
-
     final GroupByQuery query = GroupByQuery
         .builder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
         .setDimensions(new DefaultDimensionSpec("quality", "alias"))
         .setAggregatorSpecs(new LongSumAggregatorFactory("rows", "rows"))
-        .setGranularity(QueryRunnerTestHelper.DAY_GRAN)
+        .setGranularity(Granularities.ALL)
         .overrideContext(ImmutableMap.of(QueryContexts.TIMEOUT_KEY, 1))
         .build();
 
@@ -288,6 +309,10 @@ public class GroupByQueryRunnerFailureTest
     };
 
     QueryRunner<ResultRow> mergeRunners = factory.mergeRunners(Execs.directExecutor(), ImmutableList.of(runner, mockRunner));
-    GroupByQueryRunnerTestHelper.runQuery(factory, mergeRunners, query);
+
+    Assert.assertThrows(
+        QueryTimeoutException.class,
+        () -> GroupByQueryRunnerTestHelper.runQuery(factory, mergeRunners, query)
+    );
   }
 }
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
index 475848edbdc..53050b20ad5 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
@@ -56,6 +56,7 @@ import org.apache.druid.query.ChainedExecutionQueryRunner;
 import org.apache.druid.query.DirectQueryProcessingPool;
 import org.apache.druid.query.DruidProcessingConfig;
 import org.apache.druid.query.FinalizeResultsQueryRunner;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.QueryDataSource;
 import org.apache.druid.query.QueryPlus;
@@ -65,6 +66,7 @@ import org.apache.druid.query.QueryToolChest;
 import org.apache.druid.query.ResourceLimitExceededException;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.TestBigDecimalSumAggregatorFactory;
+import org.apache.druid.query.TestQueryRunner;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory;
@@ -128,6 +130,9 @@ import org.apache.druid.query.lookup.LookupExtractionFn;
 import org.apache.druid.query.ordering.StringComparators;
 import org.apache.druid.query.search.ContainsSearchQuerySpec;
 import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
@@ -141,6 +146,7 @@ import org.joda.time.DateTimeZone;
 import org.joda.time.Period;
 import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Rule;
@@ -199,7 +205,7 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   private static TestGroupByBuffers BUFFER_POOLS = null;
 
   private final QueryRunner<ResultRow> runner;
-  private final QueryRunner<ResultRow> originalRunner;
+  private final TestQueryRunner<ResultRow> originalRunner;
   private final GroupByQueryRunnerFactory factory;
   private final GroupByQueryConfig config;
   private final boolean vectorize;
@@ -379,7 +385,7 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
     final List<Object[]> constructors = new ArrayList<>();
     for (GroupByQueryConfig config : testConfigs()) {
       final GroupByQueryRunnerFactory factory = makeQueryRunnerFactory(config, BUFFER_POOLS);
-      for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(factory)) {
+      for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(factory, true)) {
         for (boolean vectorize : ImmutableList.of(false, true)) {
           final String testName = StringUtils.format("config=%s, runner=%s, vectorize=%s", config, runner, vectorize);
 
@@ -414,7 +420,7 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
       String testName,
       GroupByQueryConfig config,
       GroupByQueryRunnerFactory factory,
-      QueryRunner runner,
+      TestQueryRunner runner,
       boolean vectorize
   )
   {
@@ -428,6 +434,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupBy()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -741,6 +750,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithStringPostAggregator()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -989,6 +1001,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithStringVirtualColumn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -1052,6 +1067,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithStringVirtualColumnVectorizable()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -1115,6 +1133,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithDurationGranularity()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -1167,6 +1188,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSortDimsFirst()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -1212,6 +1236,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByNoAggregators()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -1248,6 +1275,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testMultiValueDimension()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to multi-value dimensions.
     cannotVectorize();
 
@@ -1278,6 +1308,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testMultiValueDimensionNotAllowed()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final String dimName = "placementish";
 
     if (!vectorize) {
@@ -2743,6 +2776,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByMaxRowsLimitContextOverride()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -2799,6 +2835,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByTimeoutContextOverride()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -2837,6 +2876,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByMaxOnDiskStorageContextOverride()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -2858,6 +2900,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testNotEnoughDiskSpaceThroughContextOverride()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -2883,6 +2928,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithOuterMaxOnDiskStorageContextOverride()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -2918,6 +2966,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithRebucketRename()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -2989,6 +3040,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSimpleRenameRetainMissingNonInjective()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -3060,6 +3114,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSimpleRenameRetainMissing()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> map = new HashMap<>();
     map.put("automotive", "automotive0");
     map.put("business", "business0");
@@ -3114,6 +3171,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSimpleRenameAndMissingString()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> map = new HashMap<>();
     map.put("automotive", "automotive0");
     map.put("business", "business0");
@@ -3163,6 +3223,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSimpleRename()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> map = new HashMap<>();
     map.put("automotive", "automotive0");
     map.put("business", "business0");
@@ -3313,6 +3376,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithFirstLast()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -3400,6 +3466,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithNoResult()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.EMPTY_INTERVAL)
@@ -3422,6 +3491,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithNullProducingDimExtractionFn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -3483,6 +3555,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
    */
   public void testGroupByWithEmptyStringProducingDimExtractionFn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final ExtractionFn emptyStringExtractionFn = new RegexDimExtractionFn("(\\w{1})", false, null)
     {
       @Override
@@ -3536,6 +3611,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithTimeZone()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     DateTimeZone tz = DateTimes.inferTzFromString("America/Los_Angeles");
 
     GroupByQuery query = makeQueryBuilder()
@@ -3746,6 +3824,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testMergeResults()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -3835,6 +3916,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
 
   private void doTestMergeResultsWithValidLimit(final int limit, final int offset)
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -3950,6 +4034,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final int limit = 14;
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
@@ -3992,6 +4079,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderByUsingMathExpressions()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final int limit = 14;
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
@@ -4056,6 +4146,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testMergeResultsWithOrderBy()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     LimitSpec[] orderBySpecs = new LimitSpec[]{
         new DefaultLimitSpec(OrderByColumnSpec.ascending("idx"), null),
         new DefaultLimitSpec(OrderByColumnSpec.ascending("rows", "idx"), null),
@@ -4150,6 +4243,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByOrderLimit()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -4248,6 +4344,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithOrderLimit2()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -4289,6 +4388,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithOrderLimit3()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -4332,6 +4434,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByOrderLimitNumeric()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -4704,6 +4809,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithAlphaNumericDimensionOrder()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -4762,6 +4870,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithLookupAndLimitAndSortByDimsFirst()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -4822,6 +4933,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   // in time when Druid does support this, we can re-evaluate this test.
   public void testLimitPerGrouping()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setGranularity(QueryRunnerTestHelper.DAY_GRAN).setDimensions(new DefaultDimensionSpec(
@@ -4860,6 +4974,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testPostAggMergedHavingSpec()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -5085,6 +5202,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testPostAggHavingSpec()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -5140,6 +5260,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testHavingSpec()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -5173,6 +5296,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testDimFilterHavingSpec()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final DimFilterHavingSpec havingSpec = new DimFilterHavingSpec(
         new AndDimFilter(
             ImmutableList.of(
@@ -5278,6 +5404,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testDimFilterHavingSpecWithExtractionFns()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     String extractionJsFn = "function(str) { return 'super-' + str; }";
     ExtractionFn extractionFn = new JavaScriptExtractionFn(
         extractionJsFn,
@@ -5328,6 +5457,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testMergedHavingSpec()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -5391,6 +5523,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testMergedPostAggHavingSpec()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -5500,6 +5635,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testCustomAggregatorHavingSpec()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -5540,6 +5678,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithRegEx()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -5565,6 +5706,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithNonexistentDimension()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -5625,6 +5769,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testIdenticalSubquery()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -5681,6 +5828,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithMultipleIntervalsInOuterQuery()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -5746,6 +5896,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   {
     //https://github.com/apache/druid/issues/2556
 
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -5791,6 +5944,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testDifferentGroupingSubquery()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -5854,6 +6010,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testDifferentGroupingSubqueryMultipleAggregatorsOnSameField()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -5903,6 +6062,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testDifferentGroupingSubqueryWithFilter()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -5945,6 +6107,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testDifferentIntervalSubquery()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -5971,7 +6136,7 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testDoubleMeanQuery()
   {
-    GroupByQuery query = new GroupByQuery.Builder()
+    GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setGranularity(Granularities.ALL)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -6057,6 +6222,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testEmptySubquery()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.EMPTY_INTERVAL)
@@ -6079,6 +6247,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithPostAggregators()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -6353,6 +6524,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithPostAggregatorsAndHaving()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -6627,6 +6801,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithMultiColumnAggregators()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to javascript functionality.
     cannotVectorize();
 
@@ -6810,6 +6987,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithOuterFilterAggregator()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -6837,6 +7017,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithOuterTimeFilter()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -6882,6 +7065,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithContextTimeout()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -6907,6 +7093,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithOuterVirtualColumns()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -6962,6 +7151,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithOuterCountAggregator()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -6992,6 +7184,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithOuterDimJavascriptAggregators()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -7043,6 +7238,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithOuterJavascriptAggregators()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     final GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -7094,6 +7292,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithHyperUniques()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -7237,6 +7438,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithHyperUniquesPostAggregator()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -7294,6 +7498,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithFirstLast()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
@@ -7332,6 +7539,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSubtotalsSpecOfDimensionsPrefixes()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -7446,6 +7656,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSubtotalsSpecGeneral()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -7802,6 +8015,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSubtotalsSpecWithRenamedDimensionAndFilter()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = GroupByQuery
         .builder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
@@ -7896,6 +8112,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSubtotalsSpecWithLongDimensionColumn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -8187,6 +8406,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSubtotalsSpecWithOrderLimit()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -8222,6 +8444,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSubtotalsSpecWithOrderLimitAndOffset()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -8257,6 +8482,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithSubtotalsSpecWithOrderLimitForcePushdown()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -8834,6 +9062,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testBySegmentResults()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04")
@@ -8894,6 +9125,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testBySegmentResultsUnOptimizedDimextraction()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04").setDimensions(new ExtractionDimensionSpec(
@@ -8961,6 +9195,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testBySegmentResultsOptimizedDimextraction()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery.Builder builder = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setInterval("2011-04-02/2011-04-04").setDimensions(new ExtractionDimensionSpec(
@@ -9030,6 +9267,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithExtractionDimFilter()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
     extractionMap.put("automotive", "automotiveAndBusinessAndNewsAndMezzanine");
     extractionMap.put("business", "automotiveAndBusinessAndNewsAndMezzanine");
@@ -9089,6 +9329,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
     extractionMap.put("automotive", "automotive0");
     extractionMap.put("business", "business0");
@@ -9138,6 +9381,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithExtractionDimFilterWhenSearchValueNotInTheMap()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
     MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false);
     LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false);
@@ -9165,6 +9411,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithExtractionDimFilterKeyisNull()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
 
 
@@ -9209,6 +9458,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithAggregatorFilterAndExtractionFunction()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
     extractionMap.put("automotive", "automotive0");
     extractionMap.put("business", "business0");
@@ -9392,6 +9644,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithExtractionDimFilterOptimazitionManyToOne()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
     extractionMap.put("mezzanine", "newsANDmezzanine");
     extractionMap.put("news", "newsANDmezzanine");
@@ -9431,6 +9686,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithInjectiveLookupDimFilterNullDimsOptimized()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
     extractionMap.put("", "EMPTY");
 
@@ -9465,6 +9723,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithInjectiveLookupDimFilterNullDimsNotOptimized()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
     extractionMap.put("", "EMPTY");
 
@@ -9499,6 +9760,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testBySegmentResultsWithAllFiltersWithExtractionFns()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     String extractionJsFn = "function(str) { return 'super-' + str; }";
     String jsFn = "function(x) { return(x === 'super-mezzanine') }";
     ExtractionFn extractionFn = new JavaScriptExtractionFn(
@@ -9593,6 +9857,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithAllFiltersOnNullDimsWithExtractionFns()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     Map<String, String> extractionMap = new HashMap<>();
     extractionMap.put("", "EMPTY");
     extractionMap.put(null, "EMPTY");
@@ -9639,6 +9906,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByCardinalityAggWithExtractionFn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -9731,6 +10001,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByCardinalityAggOnFloat()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -9922,6 +10195,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByLongColumn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -10033,6 +10309,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByLongColumnWithExFn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -10077,6 +10356,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByLongTimeColumn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -10115,6 +10397,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByLongTimeColumnWithExFn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -10159,6 +10444,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByFloatColumn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -10295,6 +10583,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByFloatColumnWithExFn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -10394,6 +10685,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByLongAndFloatOutputAsString()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -10439,6 +10733,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByNumericStringsAsNumeric()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -10609,6 +10906,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByNestedWithInnerQueryNumerics()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -10680,6 +10980,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByNestedWithInnerQueryOutputNullNumerics()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     cannotVectorize();
 
     // Following extractionFn will generate null value for one kind of quality
@@ -10844,6 +11147,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByStringOutputAsLong()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -10893,6 +11199,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithAggsOnNumericDimensions()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to javascript aggregators.
     cannotVectorize();
 
@@ -10961,6 +11270,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByNestedOuterExtractionFnOnFloatInner()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -11004,6 +11316,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByNestedDoubleTimeExtractionFnWithLongOutputTypes()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -11677,6 +11992,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testEmptySubqueryWithLimitPushDown()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.EMPTY_INTERVAL)
@@ -11710,6 +12028,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSubqueryWithMultipleIntervalsInOuterQueryWithLimitPushDown()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -11781,6 +12102,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testVirtualColumnFilterOnInnerQuery()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -11878,6 +12202,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testTypeConversionWithMergingChainedExecutionRunner()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // Cannot vectorize due to extraction dimension spec.
     cannotVectorize();
 
@@ -11915,6 +12242,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByNoMatchingPrefilter()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -12171,8 +12501,12 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithExpressionAggregator()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // expression agg not yet vectorized
     cannotVectorize();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -12494,6 +12828,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByWithExpressionAggregatorWithArrays()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // expression agg not yet vectorized
     cannotVectorize();
 
@@ -12781,6 +13118,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testGroupByExpressionAggregatorArrayMultiValue()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     // expression agg not yet vectorized
     cannotVectorize();
 
@@ -13201,6 +13541,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSummaryrowForEmptySubqueryInput()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery subquery = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -13250,6 +13593,9 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testSummaryrowForEmptyInputByDay()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     GroupByQuery query = makeQueryBuilder()
         .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
@@ -13313,6 +13659,14 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
     return new GroupByQuery.Builder(query).overrideContext(makeContext());
   }
 
+  private void assumeTimeOrdered()
+  {
+    try (final CursorHolder cursorHolder =
+             originalRunner.getSegment().asStorageAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Assume.assumeTrue(Cursors.getTimeOrdering(cursorHolder.getOrdering()) == Order.ASCENDING);
+    }
+  }
+
   private Map<String, Object> makeContext()
   {
     return ImmutableMap.<String, Object>builder()
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByTimeseriesQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByTimeseriesQueryRunnerTest.java
index d8a609800bb..1309334a939 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByTimeseriesQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByTimeseriesQueryRunnerTest.java
@@ -100,7 +100,7 @@ public class GroupByTimeseriesQueryRunnerTest extends TimeseriesQueryRunnerTest
 
     final List<Object[]> constructors = new ArrayList<>();
 
-    for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunnersToMerge(factory)) {
+    for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunnersToMerge(factory, false)) {
       final QueryRunner modifiedRunner = new QueryRunner()
       {
         @Override
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngineIteratorTest.java b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngineIteratorTest.java
index ce38427728c..15982c2caab 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngineIteratorTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngineIteratorTest.java
@@ -33,6 +33,7 @@ import org.apache.druid.query.groupby.epinephelinae.vector.VectorGroupByEngine.V
 import org.apache.druid.segment.ColumnProcessors;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexTimeBoundaryInspector;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.TestIndex;
 import org.apache.druid.segment.vector.VectorCursor;
@@ -62,6 +63,8 @@ public class VectorGroupByEngineIteratorTest extends InitializedNullHandlingTest
         .setAggregatorSpecs(factory)
         .build();
     final StorageAdapter storageAdapter = new QueryableIndexStorageAdapter(TestIndex.getMMappedTestIndex());
+    final QueryableIndexTimeBoundaryInspector timeBoundaryInspector =
+        QueryableIndexTimeBoundaryInspector.create(TestIndex.getMMappedTestIndex());
     final CursorHolder cursorHolder = storageAdapter.makeCursorHolder(
         GroupingEngine.makeCursorBuildSpec(query, null)
     );
@@ -81,7 +84,9 @@ public class VectorGroupByEngineIteratorTest extends InitializedNullHandlingTest
         new GroupByQueryConfig(),
         GroupByQueryRunnerTest.DEFAULT_PROCESSING_CONFIG,
         storageAdapter,
+        timeBoundaryInspector,
         cursor,
+        cursorHolder.getTimeOrder(),
         interval,
         dimensions,
         byteBuffer,
diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java
index 503eedb5db8..8a5605bc673 100644
--- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java
@@ -45,6 +45,7 @@ import org.apache.druid.query.metadata.metadata.SegmentAnalysis;
 import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
 import org.apache.druid.query.spec.LegacySegmentSpec;
 import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.IncrementalIndexSegment;
 import org.apache.druid.segment.IndexBuilder;
 import org.apache.druid.segment.QueryableIndex;
@@ -438,6 +439,9 @@ public class SegmentAnalyzerTest extends InitializedNullHandlingTest
     EasyMock.expect(mockIndex.getColumnCapabilities("x"))
             .andReturn(ColumnCapabilitiesImpl.createDefault().setType(ColumnType.UNKNOWN_COMPLEX))
             .atLeastOnce();
+    EasyMock.expect(mockIndex.getOrdering())
+            .andReturn(Cursors.ascendingTimeOrder())
+            .atLeastOnce();
 
     ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
     EasyMock.expect(mockIndex.getColumnHolder("x")).andReturn(holder).atLeastOnce();
diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java
index b1db91a9cb8..33147ce01b0 100644
--- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java
@@ -24,9 +24,9 @@ import com.google.common.collect.ImmutableMap;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.QueryPlus;
-import org.apache.druid.query.QueryRunner;
 import org.apache.druid.query.QueryRunnerFactory;
 import org.apache.druid.query.QueryRunnerTestHelper;
+import org.apache.druid.query.TestQueryRunner;
 import org.apache.druid.query.metadata.metadata.ColumnAnalysis;
 import org.apache.druid.query.metadata.metadata.ListColumnIncluderator;
 import org.apache.druid.query.metadata.metadata.SegmentAnalysis;
@@ -57,11 +57,11 @@ public class SegmentMetadataUnionQueryTest extends InitializedNullHandlingTest
           QueryRunnerTestHelper.NOOP_QUERYWATCHER
       );
 
-  private final QueryRunner<SegmentAnalysis> runner;
+  private final TestQueryRunner<SegmentAnalysis> runner;
   private final boolean mmap;
 
   public SegmentMetadataUnionQueryTest(
-      QueryRunner<SegmentAnalysis> runner,
+      TestQueryRunner<SegmentAnalysis> runner,
       boolean mmap
   )
   {
@@ -72,7 +72,7 @@ public class SegmentMetadataUnionQueryTest extends InitializedNullHandlingTest
   @Parameterized.Parameters
   public static Iterable<Object[]> constructorFeeder()
   {
-    final ArrayList<QueryRunner<SegmentAnalysis>> runners = QueryRunnerTestHelper.mapQueryRunnersToMerge(
+    final ArrayList<TestQueryRunner<SegmentAnalysis>> runners = QueryRunnerTestHelper.mapQueryRunnersToMerge(
         FACTORY,
         ImmutableList.of(
             QueryRunnerTestHelper.makeQueryRunner(
diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerTest.java
index 8dd18390618..7693f05471f 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryRunnerTest.java
@@ -151,9 +151,7 @@ public class ScanQueryRunnerTest extends InitializedNullHandlingTest
   public static Iterable<Object[]> constructorFeeder()
   {
     return Iterables.transform(
-        QueryRunnerTestHelper.makeQueryRunners(
-            FACTORY
-        ),
+        QueryRunnerTestHelper.makeQueryRunners(FACTORY, false),
         (runner) -> new Object[]{runner}
     );
   }
diff --git a/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java
index 306ed04f634..486cfb95017 100644
--- a/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java
@@ -94,7 +94,8 @@ public class SearchQueryRunnerTest extends InitializedNullHandlingTest
                 SELECTOR,
                 TOOL_CHEST,
                 QueryRunnerTestHelper.NOOP_QUERYWATCHER
-            )
+            ),
+            true
         )
     );
   }
diff --git a/processing/src/test/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java
index 02ba0ac44f4..6b1ee55568f 100644
--- a/processing/src/test/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java
@@ -37,6 +37,7 @@ import org.apache.druid.query.QueryRunnerFactory;
 import org.apache.druid.query.QueryRunnerTestHelper;
 import org.apache.druid.query.Result;
 import org.apache.druid.query.TableDataSource;
+import org.apache.druid.query.TestQueryRunner;
 import org.apache.druid.query.context.ConcurrentResponseContext;
 import org.apache.druid.query.context.ResponseContext;
 import org.apache.druid.query.ordering.StringComparators;
@@ -79,12 +80,13 @@ public class TimeBoundaryQueryRunnerTest extends InitializedNullHandlingTest
   {
     return QueryRunnerTestHelper.transformToConstructionFeeder(
         QueryRunnerTestHelper.makeQueryRunners(
-            new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER)
+            new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER),
+            true
         )
     );
   }
 
-  private final QueryRunner runner;
+  private final TestQueryRunner<Result<TimeBoundaryResultValue>> runner;
   private static final QueryRunnerFactory FACTORY = new TimeBoundaryQueryRunnerFactory(
       QueryRunnerTestHelper.NOOP_QUERYWATCHER
   );
@@ -92,7 +94,7 @@ public class TimeBoundaryQueryRunnerTest extends InitializedNullHandlingTest
   private static Segment segment1;
 
   public TimeBoundaryQueryRunnerTest(
-      QueryRunner runner
+      TestQueryRunner<Result<TimeBoundaryResultValue>> runner
   )
   {
     this.runner = runner;
diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java
index 7118b2ca50e..4cca90c083d 100644
--- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java
@@ -65,7 +65,8 @@ public class TimeSeriesUnionQueryRunnerTest extends InitializedNullHandlingTest
                 new TimeseriesQueryQueryToolChest(),
                 new TimeseriesQueryEngine(),
                 QueryRunnerTestHelper.NOOP_QUERYWATCHER
-            )
+            ),
+            false
         ),
         // descending?
         Arrays.asList(false, true)
diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java
index eedbdf8410b..781d9d31023 100644
--- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java
@@ -115,7 +115,8 @@ public class TimeseriesQueryRunnerTest extends InitializedNullHandlingTest
                 new TimeseriesQueryQueryToolChest(),
                 new TimeseriesQueryEngine(),
                 QueryRunnerTestHelper.NOOP_QUERYWATCHER
-            )
+            ),
+            false
         ),
         // descending?
         Arrays.asList(false, true),
diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNMetricSpecOptimizationsTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNMetricSpecOptimizationsTest.java
index 5c5cba6ece6..ae320ac4261 100644
--- a/processing/src/test/java/org/apache/druid/query/topn/TopNMetricSpecOptimizationsTest.java
+++ b/processing/src/test/java/org/apache/druid/query/topn/TopNMetricSpecOptimizationsTest.java
@@ -21,7 +21,6 @@ package org.apache.druid.query.topn;
 
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
-import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.query.QueryRunnerTestHelper;
@@ -41,7 +40,6 @@ import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.IndexedInts;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
 import org.junit.Test;
@@ -251,19 +249,6 @@ public class TopNMetricSpecOptimizationsTest
         return cardinality;
       }
 
-      @Override
-      public DateTime getMinTime()
-      {
-        return DateTimes.of(start);
-      }
-
-
-      @Override
-      public DateTime getMaxTime()
-      {
-        return DateTimes.of(end);
-      }
-
       // stubs below this line not important for tests
 
       @Override
@@ -305,12 +290,6 @@ public class TopNMetricSpecOptimizationsTest
         return 0;
       }
 
-      @Override
-      public DateTime getMaxIngestedEventTime()
-      {
-        return null;
-      }
-
       @Override
       public Metadata getMetadata()
       {
diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java
index b0f015c627f..c674e575d70 100644
--- a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java
@@ -42,11 +42,13 @@ import org.apache.druid.js.JavaScriptConfig;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.BySegmentResultValue;
 import org.apache.druid.query.BySegmentResultValueClass;
+import org.apache.druid.query.Order;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.QueryPlus;
 import org.apache.druid.query.QueryRunner;
 import org.apache.druid.query.QueryRunnerTestHelper;
 import org.apache.druid.query.Result;
+import org.apache.druid.query.TestQueryRunner;
 import org.apache.druid.query.TestQueryRunners;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
@@ -89,6 +91,9 @@ import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.query.lookup.LookupExtractionFn;
 import org.apache.druid.query.ordering.StringComparators;
 import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
@@ -96,6 +101,7 @@ import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -129,7 +135,7 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
   @Parameterized.Parameters(name = "{7}")
   public static Iterable<Object[]> constructorFeeder()
   {
-    List<QueryRunner<Result<TopNResultValue>>> retVal = queryRunners();
+    List<QueryRunner<Result<TopNResultValue>>> retVal = queryRunners(true);
     List<Object[]> parameters = new ArrayList<>();
     for (int i = 0; i < 32; i++) {
       for (QueryRunner<Result<TopNResultValue>> firstParameter : retVal) {
@@ -152,7 +158,13 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
     return parameters;
   }
 
-  public static List<QueryRunner<Result<TopNResultValue>>> queryRunners()
+  /**
+   * Create test query runners.
+   *
+   * @param includeNonTimeOrdered whether to include runners with non-time-ordered segments. Some test suites are not
+   *                              compatible with non-time-ordered segments.
+   */
+  public static List<QueryRunner<Result<TopNResultValue>>> queryRunners(boolean includeNonTimeOrdered)
   {
     final CloseableStupidPool<ByteBuffer> defaultPool = TestQueryRunners.createDefaultNonBlockingPool();
     final CloseableStupidPool<ByteBuffer> customPool = new CloseableStupidPool<>(
@@ -167,7 +179,8 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
                 defaultPool,
                 new TopNQueryQueryToolChest(new TopNQueryConfig()),
                 QueryRunnerTestHelper.NOOP_QUERYWATCHER
-            )
+            ),
+            includeNonTimeOrdered
         )
     );
     retVal.addAll(
@@ -176,7 +189,8 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
                 customPool,
                 new TopNQueryQueryToolChest(new TopNQueryConfig()),
                 QueryRunnerTestHelper.NOOP_QUERYWATCHER
-            )
+            ),
+            includeNonTimeOrdered
         )
     );
 
@@ -191,7 +205,7 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
     return retVal;
   }
 
-  private final QueryRunner<Result<TopNResultValue>> runner;
+  private final TestQueryRunner<Result<TopNResultValue>> runner;
   private final boolean duplicateSingleAggregatorQueries;
   private final List<AggregatorFactory> commonAggregators;
 
@@ -201,7 +215,7 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
 
   @SuppressWarnings("unused")
   public TopNQueryRunnerTest(
-      QueryRunner<Result<TopNResultValue>> runner,
+      TestQueryRunner<Result<TopNResultValue>> runner,
       boolean specializeGeneric1AggPooledTopN,
       boolean specializeGeneric2AggPooledTopN,
       boolean specializeHistorical1SimpleDoubleAggPooledTopN,
@@ -819,6 +833,9 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testTopNOverFirstLastAggregator()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     TopNQuery query = new TopNQueryBuilder()
         .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .granularity(QueryRunnerTestHelper.MONTH_GRAN)
@@ -928,6 +945,9 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testTopNOverFirstLastFloatAggregatorUsingDoubleColumn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     TopNQuery query = new TopNQueryBuilder()
         .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .granularity(QueryRunnerTestHelper.MONTH_GRAN)
@@ -1037,6 +1057,9 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testTopNOverFirstLastFloatAggregatorUsingFloatColumn()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     TopNQuery query = new TopNQueryBuilder()
         .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .granularity(QueryRunnerTestHelper.MONTH_GRAN)
@@ -6245,6 +6268,9 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
   @Test
   public void testTopNAggregateTopnMetricFirstWithGranularity()
   {
+    // Granularity != ALL requires time-ordering.
+    assumeTimeOrdered();
+
     TopNQuery query = new TopNQueryBuilder()
         .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
         .granularity(QueryRunnerTestHelper.MONTH_GRAN)
@@ -6352,6 +6378,14 @@ public class TopNQueryRunnerTest extends InitializedNullHandlingTest
     assertExpectedResults(expectedResults, query);
   }
 
+  private void assumeTimeOrdered()
+  {
+    try (final CursorHolder cursorHolder =
+             runner.getSegment().asStorageAdapter().makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+      Assume.assumeTrue(Cursors.getTimeOrdering(cursorHolder.getOrdering()) == Order.ASCENDING);
+    }
+  }
+
   private static Map<String, Object> makeRowWithNulls(
       String dimName,
       @Nullable Object dimValue,
diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNUnionQueryTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNUnionQueryTest.java
index 29f6a2c012b..031d3209a84 100644
--- a/processing/src/test/java/org/apache/druid/query/topn/TopNUnionQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/topn/TopNUnionQueryTest.java
@@ -57,7 +57,7 @@ public class TopNUnionQueryTest extends InitializedNullHandlingTest
   @Parameterized.Parameters(name = "{0}")
   public static Iterable<Object[]> constructorFeeder()
   {
-    return QueryRunnerTestHelper.transformToConstructionFeeder(TopNQueryRunnerTest.queryRunners());
+    return QueryRunnerTestHelper.transformToConstructionFeeder(TopNQueryRunnerTest.queryRunners(true));
   }
 
   private final QueryRunner runner;
diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerLongestSharedDimOrderTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerLongestSharedDimOrderTest.java
index 1fedf260553..9d8afc2d6e8 100644
--- a/processing/src/test/java/org/apache/druid/segment/IndexMergerLongestSharedDimOrderTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerLongestSharedDimOrderTest.java
@@ -78,14 +78,14 @@ public class IndexMergerLongestSharedDimOrderTest
     QueryableIndexIndexableAdapter index2 = makeIndexWithDimensionList(ImmutableList.of("b", "c"));
     List<String> actual = IndexMerger.getLongestSharedDimOrder(ImmutableList.of(index1, index2), null);
     Assert.assertNotNull(actual);
-    Assert.assertEquals(ImmutableList.of("a", "b", "c"), actual);
+    Assert.assertEquals(ImmutableList.of("__time", "a", "b", "c"), actual);
 
     //  Valid ordering as although second index has gap, it is still same ordering
     index1 = makeIndexWithDimensionList(ImmutableList.of("a", "b", "c"));
     index2 = makeIndexWithDimensionList(ImmutableList.of("a", "c"));
     actual = IndexMerger.getLongestSharedDimOrder(ImmutableList.of(index1, index2), null);
     Assert.assertNotNull(actual);
-    Assert.assertEquals(ImmutableList.of("a", "b", "c"), actual);
+    Assert.assertEquals(ImmutableList.of("__time", "a", "b", "c"), actual);
   }
 
   @Test
@@ -131,7 +131,7 @@ public class IndexMergerLongestSharedDimOrderTest
     QueryableIndexIndexableAdapter index2 = makeIndexWithDimensionList(ImmutableList.of("b", "c"));
     List<String> actual = IndexMerger.getLongestSharedDimOrder(ImmutableList.of(index1, index2), valid);
     Assert.assertNotNull(actual);
-    Assert.assertEquals(ImmutableList.of("a", "b", "c"), actual);
+    Assert.assertEquals(ImmutableList.of("__time", "a", "b", "c"), actual);
   }
 
   @Test
diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
index 7f1e227f1ec..1f46534266f 100644
--- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
+++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
@@ -37,11 +37,13 @@ import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.FloatDimensionSchema;
 import org.apache.druid.data.input.impl.LongDimensionSchema;
 import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.frame.testutil.FrameTestUtil;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper;
 import org.apache.druid.query.DefaultBitmapResultFactory;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@@ -152,7 +154,9 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
   )
   {
     this.indexSpec = IndexSpec.builder()
-                              .withBitmapSerdeFactory(bitmapSerdeFactory != null ? bitmapSerdeFactory : new ConciseBitmapSerdeFactory())
+                              .withBitmapSerdeFactory(bitmapSerdeFactory != null
+                                                      ? bitmapSerdeFactory
+                                                      : new ConciseBitmapSerdeFactory())
                               .withDimensionCompression(dimCompressionStrategy)
                               .withMetricCompression(compressionStrategy)
                               .withLongEncoding(longEncodingStrategy)
@@ -176,6 +180,7 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
 
     Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
     Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("__time"), Lists.newArrayList(index.getOrdering()));
     Assert.assertEquals(3, index.getColumnNames().size());
 
     assertDimCompression(index, indexSpec.getDimensionCompression());
@@ -191,6 +196,144 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     );
   }
 
+  @Test
+  public void testPersistPlainNonTimeOrdered() throws Exception
+  {
+    final long timestamp = System.currentTimeMillis();
+
+    IncrementalIndex toPersist =
+        new OnheapIncrementalIndex.Builder()
+            .setIndexSchema(
+                new IncrementalIndexSchema.Builder()
+                    .withDimensionsSpec(
+                        DimensionsSpec.builder()
+                                      .setDimensions(
+                                          ImmutableList.of(
+                                              new StringDimensionSchema("dim1"),
+                                              new StringDimensionSchema("dim2"),
+                                              new LongDimensionSchema("__time")
+                                          )
+                                      )
+                                      .setForceSegmentSortByTime(false)
+                                      .build()
+                    )
+                    .withMetrics(new CountAggregatorFactory("count"))
+                    .withQueryGranularity(Granularities.NONE)
+                    .withRollup(false)
+                    .build()
+            )
+            .setMaxRowCount(1000000)
+            .build();
+
+    // Expect 6 rows (no rollup).
+    IncrementalIndexTest.populateIndex(timestamp, toPersist);
+    IncrementalIndexTest.populateIndex(timestamp, toPersist);
+    IncrementalIndexTest.populateIndex(timestamp + 1, toPersist);
+
+    final File tempDir = temporaryFolder.newFolder();
+    QueryableIndex index = closer.closeLater(
+        indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))
+    );
+
+    Assert.assertEquals(6, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
+    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("dim1", "dim2", "__time"), Lists.newArrayList(index.getOrdering()));
+    Assert.assertEquals(3, index.getColumnNames().size());
+
+    assertDimCompression(index, indexSpec.getDimensionCompression());
+
+    Assert.assertArrayEquals(
+        IncrementalIndexTest.getDefaultCombiningAggregatorFactories(),
+        index.getMetadata().getAggregators()
+    );
+
+    Assert.assertEquals(
+        Granularities.NONE,
+        index.getMetadata().getQueryGranularity()
+    );
+
+    Assert.assertEquals(6, index.getNumRows());
+    Assert.assertEquals(
+        ImmutableList.of(
+            ImmutableList.of(timestamp, "1", "2", 1L),
+            ImmutableList.of(timestamp, "1", "2", 1L),
+            ImmutableList.of(timestamp + 1, "1", "2", 1L),
+            ImmutableList.of(timestamp, "3", "4", 1L),
+            ImmutableList.of(timestamp, "3", "4", 1L),
+            ImmutableList.of(timestamp + 1, "3", "4", 1L)
+        ),
+        FrameTestUtil.readRowsFromAdapter(new QueryableIndexStorageAdapter(index), null, false).toList()
+    );
+  }
+
+  @Test
+  public void testPersistRollupNonTimeOrdered() throws Exception
+  {
+    final long timestamp = System.currentTimeMillis();
+
+    IncrementalIndex toPersist =
+        new OnheapIncrementalIndex.Builder()
+            .setIndexSchema(
+                new IncrementalIndexSchema.Builder()
+                    .withDimensionsSpec(
+                        DimensionsSpec.builder()
+                                      .setDimensions(
+                                          ImmutableList.of(
+                                              new StringDimensionSchema("dim1"),
+                                              new StringDimensionSchema("dim2"),
+                                              new LongDimensionSchema("__time")
+                                          )
+                                      )
+                                      .setForceSegmentSortByTime(false)
+                                      .build()
+                    )
+                    .withMetrics(new CountAggregatorFactory("count"))
+                    .withQueryGranularity(Granularities.NONE)
+                    .withRollup(true)
+                    .build()
+            )
+            .setMaxRowCount(1000000)
+            .build();
+
+    // Expect 4 rows: the first two calls to populateIndex should roll up; the third call will create distinct rows.
+    IncrementalIndexTest.populateIndex(timestamp, toPersist);
+    IncrementalIndexTest.populateIndex(timestamp, toPersist);
+    IncrementalIndexTest.populateIndex(timestamp + 1, toPersist);
+
+    final File tempDir = temporaryFolder.newFolder();
+    QueryableIndex index = closer.closeLater(
+        indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))
+    );
+
+    Assert.assertEquals(4, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
+    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("dim1", "dim2", "__time"), Lists.newArrayList(index.getOrdering()));
+    Assert.assertEquals(3, index.getColumnNames().size());
+
+    assertDimCompression(index, indexSpec.getDimensionCompression());
+
+    Assert.assertArrayEquals(
+        IncrementalIndexTest.getDefaultCombiningAggregatorFactories(),
+        index.getMetadata().getAggregators()
+    );
+
+    Assert.assertEquals(
+        Granularities.NONE,
+        index.getMetadata().getQueryGranularity()
+    );
+
+    Assert.assertEquals(4, index.getNumRows());
+    Assert.assertEquals(
+        ImmutableList.of(
+            ImmutableList.of(timestamp, "1", "2", 2L),
+            ImmutableList.of(timestamp + 1, "1", "2", 1L),
+            ImmutableList.of(timestamp, "3", "4", 2L),
+            ImmutableList.of(timestamp + 1, "3", "4", 1L)
+        ),
+        FrameTestUtil.readRowsFromAdapter(new QueryableIndexStorageAdapter(index), null, false).toList()
+    );
+  }
+
   @Test
   public void testPersistWithDifferentDims() throws Exception
   {
@@ -217,6 +360,7 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
 
     Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
     Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("__time"), Lists.newArrayList(index.getOrdering()));
     Assert.assertEquals(3, index.getColumnNames().size());
     assertDimCompression(index, indexSpec.getDimensionCompression());
 
@@ -253,6 +397,7 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
 
     Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
     Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("__time"), Lists.newArrayList(index.getOrdering()));
     Assert.assertEquals(3, index.getColumnNames().size());
 
     assertDimCompression(index, indexSpec.getDimensionCompression());
@@ -263,7 +408,8 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
             IncrementalIndexTest.getDefaultCombiningAggregatorFactories(),
             null,
             Granularities.NONE,
-            Boolean.TRUE
+            Boolean.TRUE,
+            Cursors.ascendingTimeOrder()
         ),
         index.getMetadata()
     );
@@ -616,7 +762,11 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
     final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
-    Assert.assertEquals(Arrays.asList("d3", "d1", "d2"), ImmutableList.copyOf(adapter.getDimensionNames()));
+    Assert.assertEquals(
+        Arrays.asList("__time", "d3", "d1", "d2"),
+        ImmutableList.copyOf(adapter.getDimensionNames(true))
+    );
+    Assert.assertEquals(Arrays.asList("d3", "d1", "d2"), ImmutableList.copyOf(adapter.getDimensionNames(false)));
     Assert.assertEquals(3, rowList.size());
 
     Assert.assertEquals(Arrays.asList("30000", "100", "4000"), rowList.get(0).dimensionValues());
@@ -706,7 +856,11 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
     final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
-    Assert.assertEquals(ImmutableList.of("dimA", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames()));
+    Assert.assertEquals(
+        ImmutableList.of("__time", "dimA", "dimC"),
+        ImmutableList.copyOf(adapter.getDimensionNames(true))
+    );
+    Assert.assertEquals(ImmutableList.of("dimA", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames(false)));
     Assert.assertEquals(4, rowList.size());
 
     Assert.assertEquals(Arrays.asList(null, "1"), rowList.get(0).dimensionValues());
@@ -739,6 +893,121 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dimB", ""));
   }
 
+  @Test
+  public void testMergePlainNonTimeOrdered() throws Exception
+  {
+    IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder()
+        .withDimensionsSpec(
+            DimensionsSpec.builder()
+                          .setDimensions(
+                              ImmutableList.of(
+                                  new StringDimensionSchema("dimA", null, useBitmapIndexes),
+                                  new StringDimensionSchema("dimB", null, useBitmapIndexes),
+                                  new StringDimensionSchema("dimC", null, useBitmapIndexes)
+                              )
+                          )
+                          .setForceSegmentSortByTime(false)
+                          .build()
+        )
+        .withMetrics(new CountAggregatorFactory("count"))
+        .withQueryGranularity(Granularities.NONE)
+        .withRollup(false)
+        .build();
+
+    IncrementalIndex toPersist1 = new OnheapIncrementalIndex.Builder()
+        .setIndexSchema(schema)
+        .setMaxRowCount(1000)
+        .build();
+    IncrementalIndex toPersist2 = new OnheapIncrementalIndex.Builder()
+        .setIndexSchema(schema)
+        .setMaxRowCount(1000)
+        .build();
+    IncrementalIndex toPersist3 = new OnheapIncrementalIndex.Builder()
+        .setIndexSchema(schema)
+        .setMaxRowCount(1000)
+        .build();
+
+    addDimValuesToIndex(toPersist1, "dimA", Arrays.asList("1", "2"));
+    addDimValuesToIndex(toPersist2, "dimA", Arrays.asList("1", "2"));
+    addDimValuesToIndex(toPersist3, "dimC", Arrays.asList("1", "2"));
+
+
+    final File tmpDir = temporaryFolder.newFolder();
+    final File tmpDir2 = temporaryFolder.newFolder();
+    final File tmpDir3 = temporaryFolder.newFolder();
+    final File tmpDirMerged = temporaryFolder.newFolder();
+
+    QueryableIndex index1 = closer.closeLater(
+        indexIO.loadIndex(indexMerger.persist(toPersist1, tmpDir, indexSpec, null))
+    );
+
+    QueryableIndex index2 = closer.closeLater(
+        indexIO.loadIndex(indexMerger.persist(toPersist2, tmpDir2, indexSpec, null))
+    );
+
+    QueryableIndex index3 = closer.closeLater(
+        indexIO.loadIndex(indexMerger.persist(toPersist3, tmpDir3, indexSpec, null))
+    );
+
+    final QueryableIndex merged = closer.closeLater(
+        indexIO.loadIndex(
+            indexMerger.mergeQueryableIndex(
+                Arrays.asList(index1, index2, index3),
+                true,
+                new AggregatorFactory[]{new CountAggregatorFactory("count")},
+                tmpDirMerged,
+                indexSpec,
+                null,
+                -1
+            )
+        )
+    );
+
+    // Confirm sort order on the merged QueryableIndex.
+    Assert.assertEquals(Arrays.asList("dimA", "dimC"), Lists.newArrayList(merged.getAvailableDimensions()));
+
+    // dimB is included even though it's not actually stored.
+    Assert.assertEquals(makeOrderBys("dimA", "dimB", "dimC", "__time"), Lists.newArrayList(merged.getOrdering()));
+
+    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
+    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
+
+    Assert.assertEquals(
+        ImmutableList.of("__time", "dimA", "dimC"),
+        ImmutableList.copyOf(adapter.getDimensionNames(true))
+    );
+    Assert.assertEquals(ImmutableList.of("dimA", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames(false)));
+    Assert.assertEquals(4, rowList.size());
+
+    Assert.assertEquals(Arrays.asList(null, "1"), rowList.get(0).dimensionValues());
+    Assert.assertEquals(Collections.singletonList(1L), rowList.get(0).metricValues());
+
+    Assert.assertEquals(Arrays.asList(null, "2"), rowList.get(1).dimensionValues());
+    Assert.assertEquals(Collections.singletonList(1L), rowList.get(1).metricValues());
+
+    Assert.assertEquals(Arrays.asList("1", null), rowList.get(2).dimensionValues());
+    Assert.assertEquals(Collections.singletonList(2L), rowList.get(2).metricValues());
+
+    Assert.assertEquals(Arrays.asList("2", null), rowList.get(3).dimensionValues());
+    Assert.assertEquals(Collections.singletonList(2L), rowList.get(3).metricValues());
+
+    Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimA").hasBitmapIndexes());
+    Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimC").hasBitmapIndexes());
+
+    if (useBitmapIndexes) {
+      checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dimA", null));
+      checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "dimA", "1"));
+      checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "dimA", "2"));
+
+      checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dimB", null));
+
+      checkBitmapIndex(Arrays.asList(2, 3), getBitmapIndex(adapter, "dimC", null));
+      checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dimC", "1"));
+      checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "dimC", "2"));
+    }
+
+    checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dimB", ""));
+  }
 
   @Test
   public void testDisjointDimMerge() throws Exception
@@ -780,7 +1049,11 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
       final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
       final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
-      Assert.assertEquals(ImmutableList.of("dimA", "dimB"), ImmutableList.copyOf(adapter.getDimensionNames()));
+      Assert.assertEquals(
+          ImmutableList.of("__time", "dimA", "dimB"),
+          ImmutableList.copyOf(adapter.getDimensionNames(true))
+      );
+      Assert.assertEquals(ImmutableList.of("dimA", "dimB"), ImmutableList.copyOf(adapter.getDimensionNames(false)));
       Assert.assertEquals(5, rowList.size());
 
       Assert.assertEquals(Arrays.asList(null, "1"), rowList.get(0).dimensionValues());
@@ -911,14 +1184,22 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
       final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
       if (NullHandling.replaceWithDefault()) {
+        Assert.assertEquals(
+            ImmutableList.of("__time", "d2", "d3", "d5", "d6", "d7", "d8", "d9"),
+            ImmutableList.copyOf(adapter.getDimensionNames(true))
+        );
         Assert.assertEquals(
             ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"),
-            ImmutableList.copyOf(adapter.getDimensionNames())
+            ImmutableList.copyOf(adapter.getDimensionNames(false))
         );
       } else {
+        Assert.assertEquals(
+            ImmutableList.of("__time", "d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"),
+            ImmutableList.copyOf(adapter.getDimensionNames(true))
+        );
         Assert.assertEquals(
             ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"),
-            ImmutableList.copyOf(adapter.getDimensionNames())
+            ImmutableList.copyOf(adapter.getDimensionNames(false))
         );
       }
       Assert.assertEquals(4, rowList.size());
@@ -1085,14 +1366,22 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
     if (NullHandling.replaceWithDefault()) {
+      Assert.assertEquals(
+          ImmutableList.of("__time", "d3", "d6", "d8", "d9"),
+          ImmutableList.copyOf(adapter.getDimensionNames(true))
+      );
       Assert.assertEquals(
           ImmutableList.of("d3", "d6", "d8", "d9"),
-          ImmutableList.copyOf(adapter.getDimensionNames())
+          ImmutableList.copyOf(adapter.getDimensionNames(false))
       );
     } else {
+      Assert.assertEquals(
+          ImmutableList.of("__time", "d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"),
+          ImmutableList.copyOf(adapter.getDimensionNames(true))
+      );
       Assert.assertEquals(
           ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"),
-          ImmutableList.copyOf(adapter.getDimensionNames())
+          ImmutableList.copyOf(adapter.getDimensionNames(false))
       );
     }
 
@@ -1232,7 +1521,11 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     final QueryableIndexIndexableAdapter adapter2 = new QueryableIndexIndexableAdapter(merged2);
     final List<DebugRow> rowList2 = RowIteratorHelper.toList(adapter2.getRows());
 
-    Assert.assertEquals(ImmutableList.of("dimB", "dimA"), ImmutableList.copyOf(adapter.getDimensionNames()));
+    Assert.assertEquals(
+        ImmutableList.of("__time", "dimB", "dimA"),
+        ImmutableList.copyOf(adapter.getDimensionNames(true))
+    );
+    Assert.assertEquals(ImmutableList.of("dimB", "dimA"), ImmutableList.copyOf(adapter.getDimensionNames(false)));
     Assert.assertEquals(5, rowList.size());
 
     Assert.assertEquals(Arrays.asList(null, "1"), rowList.get(0).dimensionValues());
@@ -1259,8 +1552,14 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "dimB", "2"));
     checkBitmapIndex(Collections.singletonList(4), getBitmapIndex(adapter, "dimB", "3"));
 
-
-    Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter2.getDimensionNames()));
+    Assert.assertEquals(
+        ImmutableList.of("__time", "dimA", "dimB", "dimC"),
+        ImmutableList.copyOf(adapter2.getDimensionNames(true))
+    );
+    Assert.assertEquals(
+        ImmutableList.of("dimA", "dimB", "dimC"),
+        ImmutableList.copyOf(adapter2.getDimensionNames(false))
+    );
     Assert.assertEquals(12, rowList2.size());
     Assert.assertEquals(Arrays.asList(null, null, "1"), rowList2.get(0).dimensionValues());
     Assert.assertEquals(Collections.singletonList(1L), rowList2.get(0).metricValues());
@@ -1630,7 +1929,14 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     final IndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
     final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
-    Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames()));
+    Assert.assertEquals(
+        ImmutableList.of("__time", "dimA", "dimB", "dimC"),
+        ImmutableList.copyOf(adapter.getDimensionNames(true))
+    );
+    Assert.assertEquals(
+        ImmutableList.of("dimA", "dimB", "dimC"),
+        ImmutableList.copyOf(adapter.getDimensionNames(false))
+    );
     Assert.assertEquals(4, rowList.size());
 
     Assert.assertEquals(
@@ -1859,6 +2165,7 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
 
     Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
     Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("__time", "dim1", "dim2"), Lists.newArrayList(index.getOrdering()));
     Assert.assertEquals(3, index.getColumnNames().size());
 
     Assert.assertEquals(2, rowList.size());
@@ -1891,6 +2198,7 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
 
     Assert.assertEquals(1, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
     Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("__time", "dim1", "dim2"), Lists.newArrayList(index.getOrdering()));
     Assert.assertEquals(3, index.getColumnNames().size());
 
     adapter = new QueryableIndexIndexableAdapter(index);
@@ -1922,6 +2230,7 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
 
     Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
     Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("__time", "dim1", "dim2"), Lists.newArrayList(index.getOrdering()));
     Assert.assertEquals(3, index.getColumnNames().size());
 
     adapter = new QueryableIndexIndexableAdapter(index);
@@ -1978,10 +2287,8 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     );
 
     Assert.assertEquals(3, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
-    Assert.assertEquals(
-        Arrays.asList("dim1", "dim2"),
-        Lists.newArrayList(index.getAvailableDimensions())
-    );
+    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(makeOrderBys("__time"), Lists.newArrayList(index.getOrdering()));
     Assert.assertEquals(3, index.getColumnNames().size());
 
     assertDimCompression(index, indexSpec.getDimensionCompression());
@@ -2087,9 +2394,13 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
     final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
+    Assert.assertEquals(
+        ImmutableList.of("__time", "dimA", "dimMultiVal"),
+        ImmutableList.copyOf(adapter.getDimensionNames(true))
+    );
     Assert.assertEquals(
         ImmutableList.of("dimA", "dimMultiVal"),
-        ImmutableList.copyOf(adapter.getDimensionNames())
+        ImmutableList.copyOf(adapter.getDimensionNames(false))
     );
 
     Assert.assertEquals(3, rowList.size());
@@ -2317,9 +2628,13 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
     final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
+    Assert.assertEquals(
+        ImmutableList.of("__time", "dimA", "dimMultiVal"),
+        ImmutableList.copyOf(adapter.getDimensionNames(true))
+    );
     Assert.assertEquals(
         ImmutableList.of("dimA", "dimMultiVal"),
-        ImmutableList.copyOf(adapter.getDimensionNames())
+        ImmutableList.copyOf(adapter.getDimensionNames(false))
     );
 
     if (NullHandling.replaceWithDefault()) {
@@ -2453,9 +2768,13 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
     final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
 
+    Assert.assertEquals(
+        ImmutableList.of("__time", "d1", "d2", "d3", "d4", "d5"),
+        ImmutableList.copyOf(adapter.getDimensionNames(true))
+    );
     Assert.assertEquals(
         ImmutableList.of("d1", "d2", "d3", "d4", "d5"),
-        ImmutableList.copyOf(adapter.getDimensionNames())
+        ImmutableList.copyOf(adapter.getDimensionNames(false))
     );
 
     Assert.assertEquals(4, rowList.size());
@@ -2725,4 +3044,9 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
                      )
                      .collect(Collectors.toList());
   }
+
+  private static List<OrderBy> makeOrderBys(final String... columnNames)
+  {
+    return Arrays.stream(columnNames).map(OrderBy::ascending).collect(Collectors.toList());
+  }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/MergingRowIteratorTest.java b/processing/src/test/java/org/apache/druid/segment/MergingRowIteratorTest.java
index 9117fb05be5..47e673f83c1 100644
--- a/processing/src/test/java/org/apache/druid/segment/MergingRowIteratorTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/MergingRowIteratorTest.java
@@ -166,6 +166,7 @@ public class MergingRowIteratorTest extends InitializedNullHandlingTest
       this.timestamps = timestamps.iterator();
       this.rowPointer = new RowPointer(
           currentTimestamp,
+          0,
           ColumnValueSelector.EMPTY_ARRAY,
           Collections.emptyList(),
           ColumnValueSelector.EMPTY_ARRAY,
@@ -174,6 +175,7 @@ public class MergingRowIteratorTest extends InitializedNullHandlingTest
       );
       this.markedRowPointer = new TimeAndDimsPointer(
           markedTimestamp,
+          0,
           ColumnValueSelector.EMPTY_ARRAY,
           Collections.emptyList(),
           ColumnValueSelector.EMPTY_ARRAY,
diff --git a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java
index c763e66b1ed..b158480b22d 100644
--- a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory;
 import org.apache.druid.query.aggregation.LongMaxAggregatorFactory;
@@ -31,10 +32,13 @@ import org.junit.Assert;
 import org.junit.Test;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
+ *
  */
 public class MetadataTest
 {
@@ -43,7 +47,7 @@ public class MetadataTest
   {
     ObjectMapper jsonMapper = TestHelper.makeJsonMapper();
 
-    AggregatorFactory[] aggregators = new AggregatorFactory[] {
+    AggregatorFactory[] aggregators = new AggregatorFactory[]{
         new LongSumAggregatorFactory("out", "in")
     };
 
@@ -52,7 +56,8 @@ public class MetadataTest
         aggregators,
         null,
         Granularities.ALL,
-        Boolean.FALSE
+        Boolean.FALSE,
+        null
     );
 
     Metadata other = jsonMapper.readValue(
@@ -75,7 +80,7 @@ public class MetadataTest
     Assert.assertNull(Metadata.merge(metadataToBeMerged, null));
 
     //sanity merge check
-    AggregatorFactory[] aggs = new AggregatorFactory[] {
+    AggregatorFactory[] aggs = new AggregatorFactory[]{
         new LongMaxAggregatorFactory("n", "f")
     };
     final Metadata m1 = new Metadata(
@@ -83,7 +88,8 @@ public class MetadataTest
         aggs,
         new TimestampSpec("ds", "auto", null),
         Granularities.ALL,
-        Boolean.FALSE
+        Boolean.FALSE,
+        null
     );
 
     final Metadata m2 = new Metadata(
@@ -91,7 +97,8 @@ public class MetadataTest
         aggs,
         new TimestampSpec("ds", "auto", null),
         Granularities.ALL,
-        Boolean.FALSE
+        Boolean.FALSE,
+        null
     );
 
     final Metadata m3 = new Metadata(
@@ -99,7 +106,8 @@ public class MetadataTest
         aggs,
         new TimestampSpec("ds", "auto", null),
         Granularities.ALL,
-        Boolean.TRUE
+        Boolean.TRUE,
+        null
     );
 
     final Metadata merged = new Metadata(
@@ -109,7 +117,8 @@ public class MetadataTest
         },
         new TimestampSpec("ds", "auto", null),
         Granularities.ALL,
-        Boolean.FALSE
+        Boolean.FALSE,
+        Cursors.ascendingTimeOrder()
     );
     Assert.assertEquals(merged, Metadata.merge(ImmutableList.of(m1, m2), null));
 
@@ -119,16 +128,18 @@ public class MetadataTest
     metadataToBeMerged.add(m2);
     metadataToBeMerged.add(null);
 
-    final Metadata merged2 = new Metadata(Collections.singletonMap("k", "v"), null, null, null, null);
+    final Metadata merged2 =
+        new Metadata(Collections.singletonMap("k", "v"), null, null, null, null, Cursors.ascendingTimeOrder());
 
     Assert.assertEquals(merged2, Metadata.merge(metadataToBeMerged, null));
 
     //merge check with client explicitly providing merged aggregators
-    AggregatorFactory[] explicitAggs = new AggregatorFactory[] {
+    AggregatorFactory[] explicitAggs = new AggregatorFactory[]{
         new DoubleMaxAggregatorFactory("x", "y")
     };
 
-    final Metadata merged3 = new Metadata(Collections.singletonMap("k", "v"), explicitAggs, null, null, null);
+    final Metadata merged3 =
+        new Metadata(Collections.singletonMap("k", "v"), explicitAggs, null, null, null, Cursors.ascendingTimeOrder());
 
     Assert.assertEquals(
         merged3,
@@ -140,11 +151,76 @@ public class MetadataTest
         explicitAggs,
         new TimestampSpec("ds", "auto", null),
         Granularities.ALL,
-        null
+        null,
+        Cursors.ascendingTimeOrder()
     );
     Assert.assertEquals(
         merged4,
         Metadata.merge(ImmutableList.of(m3, m2), explicitAggs)
     );
   }
+
+  @Test
+  public void testMergeOrderings()
+  {
+    Assert.assertThrows(
+        IllegalArgumentException.class,
+        () -> Metadata.mergeOrderings(Collections.emptyList())
+    );
+
+    Assert.assertEquals(
+        Cursors.ascendingTimeOrder(),
+        Metadata.mergeOrderings(Collections.singletonList(null))
+    );
+
+    Assert.assertEquals(
+        Collections.emptyList(),
+        Metadata.mergeOrderings(Arrays.asList(null, makeOrderBy("foo", "bar")))
+    );
+
+    Assert.assertEquals(
+        Collections.emptyList(),
+        Metadata.mergeOrderings(Arrays.asList(makeOrderBy("foo", "bar"), null))
+    );
+
+    Assert.assertEquals(
+        Cursors.ascendingTimeOrder(),
+        Metadata.mergeOrderings(Arrays.asList(makeOrderBy("__time", "foo", "bar"), null))
+    );
+
+    Assert.assertEquals(
+        Collections.emptyList(),
+        Metadata.mergeOrderings(
+            Arrays.asList(
+                makeOrderBy("foo", "bar"),
+                makeOrderBy("bar", "foo")
+            )
+        )
+    );
+
+    Assert.assertEquals(
+        Collections.singletonList(OrderBy.ascending("bar")),
+        Metadata.mergeOrderings(
+            Arrays.asList(
+                makeOrderBy("bar", "baz"),
+                makeOrderBy("bar", "foo")
+            )
+        )
+    );
+
+    Assert.assertEquals(
+        ImmutableList.of(OrderBy.ascending("bar"), OrderBy.ascending("foo")),
+        Metadata.mergeOrderings(
+            Arrays.asList(
+                makeOrderBy("bar", "foo"),
+                makeOrderBy("bar", "foo")
+            )
+        )
+    );
+  }
+
+  private static List<OrderBy> makeOrderBy(final String... columnNames)
+  {
+    return Arrays.stream(columnNames).map(OrderBy::ascending).collect(Collectors.toList());
+  }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java
index dca89de18b1..6c46050c697 100644
--- a/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java
@@ -23,6 +23,7 @@ import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.math.LongMath;
+import junitparams.converters.Nullable;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.common.guava.GuavaUtils;
 import org.apache.druid.java.util.common.DateTimes;
@@ -322,20 +323,6 @@ public class RowBasedStorageAdapterTest
     Assert.assertEquals(DimensionDictionarySelector.CARDINALITY_UNKNOWN, adapter.getDimensionCardinality("__time"));
   }
 
-  @Test
-  public void test_getMinTime()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
-    Assert.assertEquals(Intervals.ETERNITY.getStart(), adapter.getMinTime());
-  }
-
-  @Test
-  public void test_getMaxTime()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
-    Assert.assertEquals(Intervals.ETERNITY.getEnd().minus(1), adapter.getMaxTime());
-  }
-
   @Test
   public void test_getMinValue()
   {
@@ -476,13 +463,6 @@ public class RowBasedStorageAdapterTest
     assertThrows(UnsupportedOperationException.class, () -> adapter.getMetadata());
   }
 
-  @Test
-  public void test_getMaxIngestedEventTime()
-  {
-    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1, 2);
-    Assert.assertEquals(Intervals.ETERNITY.getEnd().minus(1), adapter.getMaxIngestedEventTime());
-  }
-
   @Test
   public void test_getMetadata()
   {
@@ -665,7 +645,6 @@ public class RowBasedStorageAdapterTest
                                                      .setInterval(Intervals.of("1970/1971"))
                                                      .build();
     try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
-      final Cursor cursor = cursorHolder.asCursor();
       Assert.assertEquals(
           ImmutableList.of(
               ImmutableList.of(DateTimes.of("1970-01-01T00"), "0"),
@@ -674,7 +653,7 @@ public class RowBasedStorageAdapterTest
               ImmutableList.of(DateTimes.of("1970-01-01T02"), "2"),
               ImmutableList.of(DateTimes.of("1970-01-01T03"), "3")
           ),
-          walkCursorGranularized(adapter, cursor, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
+          walkCursorGranularized(cursorHolder, null, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
       );
     }
 
@@ -691,15 +670,13 @@ public class RowBasedStorageAdapterTest
                                                      .build();
 
     try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
-      final Cursor cursor = cursorHolder.asCursor();
-
       Assert.assertEquals(
           ImmutableList.of(
               ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
               ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
               ImmutableList.of(DateTimes.of("1970-01-01T02"), "2")
           ),
-          walkCursorGranularized(adapter, cursor, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
+          walkCursorGranularized(cursorHolder, null, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
       );
     }
 
@@ -717,14 +694,13 @@ public class RowBasedStorageAdapterTest
                                                      .build();
 
     try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
-      final Cursor cursor = cursorHolder.asCursor();
       Assert.assertEquals(
           ImmutableList.of(
               ImmutableList.of(DateTimes.of("1970-01-01T02"), "2"),
               ImmutableList.of(DateTimes.of("1970-01-01T01"), "1"),
               ImmutableList.of(DateTimes.of("1970-01-01T01"), "1")
           ),
-          walkCursorGranularized(adapter, cursor, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
+          walkCursorGranularized(cursorHolder, null, buildSpec, Granularities.HOUR, READ_TIME_AND_STRING_GRAN)
       );
     }
 
@@ -890,19 +866,21 @@ public class RowBasedStorageAdapterTest
   }
 
   private static List<List<Object>> walkCursorGranularized(
-      final StorageAdapter adapter,
-      final Cursor cursor,
+      final CursorHolder cursorHolder,
+      @Nullable final TimeBoundaryInspector timeBoundaryInspector,
       final CursorBuildSpec buildSpec,
       final Granularity granularity,
       final List<BiFunction<Cursor, CursorGranularizer, Supplier<Object>>> processors
   )
   {
+    final Cursor cursor = cursorHolder.asCursor();
+
     CursorGranularizer granularizer = CursorGranularizer.create(
-        adapter,
         cursor,
+        timeBoundaryInspector,
+        Cursors.getTimeOrdering(cursorHolder.getOrdering()),
         granularity,
-        buildSpec.getInterval(),
-        Cursors.preferDescendingTimeOrdering(buildSpec)
+        buildSpec.getInterval()
     );
 
     final List<Supplier<Object>> suppliers = new ArrayList<>();
diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java
index 4aceafa5f3a..1657a77b209 100644
--- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java
+++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java
@@ -111,6 +111,23 @@ public class TestIndex
       new StringDimensionSchema("null_column")
   );
 
+  public static final List<DimensionSchema> DIMENSION_SCHEMAS_NON_TIME_ORDERED = Arrays.asList(
+      new StringDimensionSchema("market"),
+      new StringDimensionSchema("quality"),
+      new LongDimensionSchema("__time"),
+      new LongDimensionSchema("qualityLong"),
+      new FloatDimensionSchema("qualityFloat"),
+      new DoubleDimensionSchema("qualityDouble"),
+      new StringDimensionSchema("qualityNumericString"),
+      new LongDimensionSchema("longNumericNull"),
+      new FloatDimensionSchema("floatNumericNull"),
+      new DoubleDimensionSchema("doubleNumericNull"),
+      new StringDimensionSchema("placement"),
+      new StringDimensionSchema("placementish"),
+      new StringDimensionSchema("partial_null_column"),
+      new StringDimensionSchema("null_column")
+  );
+
   public static final List<DimensionSchema> DIMENSION_SCHEMAS_NO_BITMAP = Arrays.asList(
       new StringDimensionSchema("market", null, false),
       new StringDimensionSchema("quality", null, false),
@@ -128,7 +145,8 @@ public class TestIndex
   );
 
   public static final DimensionsSpec DIMENSIONS_SPEC = new DimensionsSpec(DIMENSION_SCHEMAS);
-
+  public static final DimensionsSpec DIMENSIONS_SPEC_NON_TIME_ORDERED =
+      new DimensionsSpec(DIMENSION_SCHEMAS_NON_TIME_ORDERED);
   public static final DimensionsSpec DIMENSIONS_SPEC_NO_BITMAPS = new DimensionsSpec(DIMENSION_SCHEMAS_NO_BITMAP);
 
   public static final String[] DOUBLE_METRICS = new String[]{"index", "indexMin", "indexMaxPlusTen"};
@@ -162,15 +180,27 @@ public class TestIndex
   private static Supplier<IncrementalIndex> realtimeIndex = Suppliers.memoize(
       () -> makeRealtimeIndex("druid.sample.numeric.tsv")
   );
+  private static Supplier<IncrementalIndex> nonTimeOrderedRealtimeIndex = Suppliers.memoize(
+      () -> makeRealtimeIndex("druid.sample.numeric.tsv", true, DIMENSIONS_SPEC_NON_TIME_ORDERED)
+  );
+  private static Supplier<IncrementalIndex> nonTimeOrderedNoRollupRealtimeIndex = Suppliers.memoize(
+      () -> makeRealtimeIndex("druid.sample.numeric.tsv", false, DIMENSIONS_SPEC_NON_TIME_ORDERED)
+  );
   private static Supplier<IncrementalIndex> noRollupRealtimeIndex = Suppliers.memoize(
       () -> makeRealtimeIndex("druid.sample.numeric.tsv", false)
   );
   private static Supplier<IncrementalIndex> noBitmapRealtimeIndex = Suppliers.memoize(
-      () -> makeRealtimeIndex("druid.sample.numeric.tsv", false, false)
+      () -> makeRealtimeIndex("druid.sample.numeric.tsv", false, DIMENSIONS_SPEC_NO_BITMAPS)
   );
   private static Supplier<QueryableIndex> mmappedIndex = Suppliers.memoize(
       () -> persistRealtimeAndLoadMMapped(realtimeIndex.get())
   );
+  private static Supplier<QueryableIndex> nonTimeOrderedMmappedIndex = Suppliers.memoize(
+      () -> persistRealtimeAndLoadMMapped(nonTimeOrderedRealtimeIndex.get())
+  );
+  private static Supplier<QueryableIndex> nonTimeOrderedNoRollupMmappedIndex = Suppliers.memoize(
+      () -> persistRealtimeAndLoadMMapped(nonTimeOrderedNoRollupRealtimeIndex.get())
+  );
   private static Supplier<QueryableIndex> noRollupMmappedIndex = Suppliers.memoize(
       () -> persistRealtimeAndLoadMMapped(noRollupRealtimeIndex.get())
   );
@@ -264,6 +294,26 @@ public class TestIndex
     return noBitmapMmappedIndex.get();
   }
 
+  public static IncrementalIndex getNonTimeOrderedRealtimeTestIndex()
+  {
+    return nonTimeOrderedRealtimeIndex.get();
+  }
+
+  public static IncrementalIndex getNonTimeOrderedNoRollupRealtimeTestIndex()
+  {
+    return nonTimeOrderedNoRollupRealtimeIndex.get();
+  }
+
+  public static QueryableIndex getNonTimeOrderedMMappedTestIndex()
+  {
+    return nonTimeOrderedMmappedIndex.get();
+  }
+
+  public static QueryableIndex getNonTimeOrderedNoRollupMMappedTestIndex()
+  {
+    return nonTimeOrderedNoRollupMmappedIndex.get();
+  }
+
   public static QueryableIndex mergedRealtimeIndex()
   {
     return mergedRealtime.get();
@@ -281,13 +331,17 @@ public class TestIndex
 
   public static IncrementalIndex makeRealtimeIndex(final String resourceFilename, boolean rollup)
   {
-    return makeRealtimeIndex(resourceFilename, rollup, true);
+    return makeRealtimeIndex(resourceFilename, rollup, DIMENSIONS_SPEC);
   }
 
-  public static IncrementalIndex makeRealtimeIndex(final String resourceFilename, boolean rollup, boolean bitmap)
+  public static IncrementalIndex makeRealtimeIndex(
+      final String resourceFilename,
+      boolean rollup,
+      DimensionsSpec dimensionsSpec
+  )
   {
     CharSource stream = getResourceCharSource(resourceFilename);
-    return makeRealtimeIndex(stream, rollup, bitmap);
+    return makeRealtimeIndex(stream, rollup, dimensionsSpec);
   }
 
   public static CharSource getResourceCharSource(final String resourceFilename)
@@ -302,15 +356,19 @@ public class TestIndex
 
   public static IncrementalIndex makeRealtimeIndex(final CharSource source)
   {
-    return makeRealtimeIndex(source, true, true);
+    return makeRealtimeIndex(source, true, DIMENSIONS_SPEC);
   }
 
-  public static IncrementalIndex makeRealtimeIndex(final CharSource source, boolean rollup, boolean bitmap)
+  public static IncrementalIndex makeRealtimeIndex(
+      final CharSource source,
+      boolean rollup,
+      DimensionsSpec dimensionsSpec
+  )
   {
     final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder()
         .withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis())
         .withTimestampSpec(new TimestampSpec("ds", "auto", null))
-        .withDimensionsSpec(bitmap ? DIMENSIONS_SPEC : DIMENSIONS_SPEC_NO_BITMAPS)
+        .withDimensionsSpec(dimensionsSpec)
         .withVirtualColumns(VIRTUAL_COLUMNS)
         .withMetrics(METRIC_AGGS)
         .withRollup(rollup)
diff --git a/processing/src/test/java/org/apache/druid/segment/TombstoneSegmentStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/TombstoneSegmentStorageAdapterTest.java
index 8950693f14f..93266894c69 100644
--- a/processing/src/test/java/org/apache/druid/segment/TombstoneSegmentStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/TombstoneSegmentStorageAdapterTest.java
@@ -21,7 +21,6 @@ package org.apache.druid.segment;
 
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.data.Indexed;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
 import org.junit.Test;
@@ -62,12 +61,6 @@ public class TombstoneSegmentStorageAdapterTest
         return 0;
       }
 
-      @Override
-      public DateTime getMaxIngestedEventTime()
-      {
-        return null;
-      }
-
       @Override
       public Indexed<String> getAvailableDimensions()
       {
@@ -86,18 +79,6 @@ public class TombstoneSegmentStorageAdapterTest
         return 0;
       }
 
-      @Override
-      public DateTime getMinTime()
-      {
-        return null;
-      }
-
-      @Override
-      public DateTime getMaxTime()
-      {
-        return null;
-      }
-
       @Nullable
       @Override
       public Comparable getMinValue(String column)
@@ -125,7 +106,6 @@ public class TombstoneSegmentStorageAdapterTest
       {
         return null;
       }
-
     };
 
     Assert.assertFalse(sa.isFromTombstone());
diff --git a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
index 8ea5791463f..96a06e3523b 100644
--- a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
@@ -22,7 +22,6 @@ package org.apache.druid.segment;
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.data.input.InputSource;
 import org.apache.druid.data.input.ResourceInputSource;
-import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequences;
@@ -169,21 +168,9 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
   {
     String colName = "multi-string1";
     for (StorageAdapter adapter : ADAPTERS) {
-      Assert.assertEquals(
-          DateTimes.of("2000-01-01T23:00:00.000Z"),
-          adapter.getMaxTime()
-      );
-      Assert.assertEquals(
-          DateTimes.of("2000-01-01T12:00:00.000Z"),
-          adapter.getMinTime()
-      );
       adapter.getColumnCapabilities(colName);
       Assert.assertEquals(adapter.getNumRows(), 0);
       Assert.assertNotNull(adapter.getMetadata());
-      Assert.assertEquals(
-          DateTimes.of("2000-01-01T23:59:59.999Z"),
-          adapter.getMaxIngestedEventTime()
-      );
       Assert.assertEquals(
           adapter.getColumnCapabilities(colName).toColumnType(),
           INCREMENTAL_INDEX_STORAGE_ADAPTER.getColumnCapabilities(colName).toColumnType()
diff --git a/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java b/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java
index 3143d2f7be7..9c0e3d8f158 100644
--- a/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java
@@ -231,7 +231,8 @@ public class IncrementalIndexTest extends InitializedNullHandlingTest
     IncrementalIndex index = indexCreator.createIndex((Object) DEFAULT_AGGREGATOR_FACTORIES);
 
     populateIndex(timestamp, index);
-    Assert.assertEquals(Arrays.asList("dim1", "dim2"), index.getDimensionNames());
+    Assert.assertEquals(Arrays.asList("__time", "dim1", "dim2"), index.getDimensionNames(true));
+    Assert.assertEquals(Arrays.asList("dim1", "dim2"), index.getDimensionNames(false));
     Assert.assertEquals(2, index.size());
 
     final Iterator<Row> rows = index.iterator();
@@ -286,7 +287,8 @@ public class IncrementalIndexTest extends InitializedNullHandlingTest
         )
     );
 
-    Assert.assertEquals(Arrays.asList("dim1", "dim2", "dim3"), index.getDimensionNames());
+    Assert.assertEquals(Arrays.asList("__time", "dim1", "dim2", "dim3"), index.getDimensionNames(true));
+    Assert.assertEquals(Arrays.asList("dim1", "dim2", "dim3"), index.getDimensionNames(false));
     Assert.assertEquals(
         Arrays.asList(
             "count",
@@ -649,7 +651,8 @@ public class IncrementalIndexTest extends InitializedNullHandlingTest
             .build()
     );
 
-    Assert.assertEquals(Arrays.asList("dim0", "dim1"), incrementalIndex.getDimensionNames());
+    Assert.assertEquals(Arrays.asList("__time", "dim0", "dim1"), incrementalIndex.getDimensionNames(true));
+    Assert.assertEquals(Arrays.asList("dim0", "dim1"), incrementalIndex.getDimensionNames(false));
   }
 
   @Test
diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexRowCompTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexRowCompTest.java
index 55191fd192b..b2bf5dd044b 100644
--- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexRowCompTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexRowCompTest.java
@@ -19,9 +19,10 @@
 
 package org.apache.druid.segment.incremental;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.druid.data.input.MapBasedInputRow;
+import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.segment.CloserRule;
 import org.apache.druid.testing.InitializedNullHandlingTest;
@@ -31,8 +32,10 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
@@ -42,19 +45,14 @@ import java.util.Map;
 @RunWith(Parameterized.class)
 public class IncrementalIndexRowCompTest extends InitializedNullHandlingTest
 {
-  public final IncrementalIndexCreator indexCreator;
+  public String indexType;
 
   @Rule
   public final CloserRule closer = new CloserRule(false);
 
-  public IncrementalIndexRowCompTest(String indexType) throws JsonProcessingException
+  public IncrementalIndexRowCompTest(String indexType)
   {
-    indexCreator = closer.closeLater(
-        new IncrementalIndexCreator(indexType, (builder, args) -> builder
-            .setSimpleTestingIndexSchema(new CountAggregatorFactory("cnt"))
-            .setMaxRowCount(1_000)
-            .build())
-    );
+    this.indexType = indexType;
   }
 
   @Parameterized.Parameters(name = "{index}: {0}")
@@ -64,10 +62,21 @@ public class IncrementalIndexRowCompTest extends InitializedNullHandlingTest
   }
 
   @Test
-  public void testBasic()
+  public void testBasic() throws IOException
   {
+    IncrementalIndexCreator indexCreator = closer.closeLater(
+        new IncrementalIndexCreator(indexType, (builder, args) -> builder
+            .setSimpleTestingIndexSchema(new CountAggregatorFactory("cnt"))
+            .setMaxRowCount(1_000)
+            .build())
+    );
     IncrementalIndex index = indexCreator.createIndex();
 
+    // Expected ordering: __time
+    Assert.assertEquals(0, index.timePosition);
+    Assert.assertEquals(ImmutableList.of("__time"), index.getDimensionNames(true));
+    Assert.assertEquals(Collections.emptyList(), index.getDimensionNames(false));
+
     long time = System.currentTimeMillis();
     IncrementalIndexRow ir1 = index.toIncrementalIndexRow(toMapRow(time, "billy", "A", "joe", "B")).getIncrementalIndexRow();
     IncrementalIndexRow ir2 = index.toIncrementalIndexRow(toMapRow(time, "billy", "A", "joe", "A")).getIncrementalIndexRow();
@@ -100,6 +109,69 @@ public class IncrementalIndexRowCompTest extends InitializedNullHandlingTest
     Assert.assertTrue(comparator.compare(ir4, ir5) > 0);
   }
 
+  @Test
+  public void testSortByDim() throws IOException
+  {
+    IncrementalIndexCreator indexCreator = closer.closeLater(
+        new IncrementalIndexCreator(
+            indexType,
+            (builder, args) -> builder
+                .setIndexSchema(
+                    IncrementalIndexSchema
+                        .builder()
+                        .withDimensionsSpec(
+                            DimensionsSpec.builder()
+                                .setDimensions(DimensionsSpec.getDefaultSchemas(Collections.singletonList("joe")))
+                                .useSchemaDiscovery(true)
+                                .setForceSegmentSortByTime(false)
+                                .build()
+                        )
+                        .withMetrics(new CountAggregatorFactory("cnt"))
+                        .build()
+                )
+                .setMaxRowCount(1_000)
+                .build()
+        )
+    );
+    IncrementalIndex index = indexCreator.createIndex();
+
+    // Expected ordering: [joe, __time]
+    Assert.assertEquals(1, index.timePosition);
+    Assert.assertEquals(ImmutableList.of("joe", "__time"), index.getDimensionNames(true));
+    Assert.assertEquals(ImmutableList.of("joe"), index.getDimensionNames(false));
+
+    long time = System.currentTimeMillis();
+    IncrementalIndexRow ir1 = index.toIncrementalIndexRow(toMapRow(time, "billy", "A", "joe", "B")).getIncrementalIndexRow();
+    IncrementalIndexRow ir2 = index.toIncrementalIndexRow(toMapRow(time, "billy", "A", "joe", "A")).getIncrementalIndexRow();
+    IncrementalIndexRow ir3 = index.toIncrementalIndexRow(toMapRow(time, "billy", "A")).getIncrementalIndexRow();
+
+    IncrementalIndexRow ir4 = index.toIncrementalIndexRow(toMapRow(time + 1, "billy", "A", "joe", "B")).getIncrementalIndexRow();
+    IncrementalIndexRow ir5 = index.toIncrementalIndexRow(toMapRow(time + 1, "billy", "A", "joe", Arrays.asList("A", "B"))).getIncrementalIndexRow();
+    IncrementalIndexRow ir6 = index.toIncrementalIndexRow(toMapRow(time + 1)).getIncrementalIndexRow();
+
+    Comparator<IncrementalIndexRow> comparator = index.dimsComparator();
+
+    Assert.assertEquals(0, comparator.compare(ir1, ir1));
+    Assert.assertEquals(0, comparator.compare(ir2, ir2));
+    Assert.assertEquals(0, comparator.compare(ir3, ir3));
+
+    Assert.assertTrue(comparator.compare(ir1, ir2) > 0);
+    Assert.assertTrue(comparator.compare(ir2, ir1) < 0);
+    Assert.assertTrue(comparator.compare(ir2, ir3) > 0);
+    Assert.assertTrue(comparator.compare(ir3, ir2) < 0);
+    Assert.assertTrue(comparator.compare(ir1, ir3) > 0);
+    Assert.assertTrue(comparator.compare(ir3, ir1) < 0);
+
+    Assert.assertTrue(comparator.compare(ir6, ir1) < 0);
+    Assert.assertTrue(comparator.compare(ir6, ir2) < 0);
+    Assert.assertTrue(comparator.compare(ir6, ir3) > 0);
+
+    Assert.assertTrue(comparator.compare(ir4, ir6) > 0);
+    Assert.assertTrue(comparator.compare(ir5, ir6) > 0);
+    Assert.assertTrue(comparator.compare(ir5, ir4) < 0);
+    Assert.assertTrue(comparator.compare(ir4, ir5) > 0);
+  }
+
   private MapBasedInputRow toMapRow(long time, Object... dimAndVal)
   {
     Map<String, Object> data = new HashMap<>();
diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
index c82003e1e84..bab827130e4 100644
--- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
@@ -20,13 +20,17 @@
 package org.apache.druid.segment.incremental;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import org.apache.druid.collections.CloseableStupidPool;
 import org.apache.druid.collections.ResourceHolder;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.MapBasedInputRow;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
 import org.apache.druid.guice.BuiltInTypesModule;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
@@ -63,6 +67,7 @@ import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.IncrementalIndexTimeBoundaryInspector;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.data.IndexedInts;
 import org.apache.druid.segment.filter.Filters;
@@ -73,6 +78,7 @@ import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -94,23 +100,56 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
 {
   public final IncrementalIndexCreator indexCreator;
 
+  /**
+   * If true, sort by [billy, __time]. If false, sort by [__time].
+   */
+  public final boolean sortByDim;
+
   @Rule
   public final CloserRule closer = new CloserRule(false);
 
-  public IncrementalIndexStorageAdapterTest(String indexType) throws JsonProcessingException
+  public IncrementalIndexStorageAdapterTest(String indexType, boolean sortByDim) throws JsonProcessingException
   {
     BuiltInTypesModule.registerHandlersAndSerde();
-    indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder
-        .setSimpleTestingIndexSchema(new CountAggregatorFactory("cnt"))
-        .setMaxRowCount(1_000)
-        .build()
-    ));
+    this.sortByDim = sortByDim;
+    this.indexCreator = closer.closeLater(
+        new IncrementalIndexCreator(
+            indexType,
+            (builder, args) -> {
+              final DimensionsSpec dimensionsSpec;
+
+              if (sortByDim) {
+                dimensionsSpec =
+                    DimensionsSpec.builder()
+                                  .setDimensions(Collections.singletonList(new StringDimensionSchema("billy")))
+                                  .setForceSegmentSortByTime(false)
+                                  .setIncludeAllDimensions(true)
+                                  .build();
+              } else {
+                dimensionsSpec = DimensionsSpec.EMPTY;
+              }
+
+              return builder
+                  .setIndexSchema(
+                      IncrementalIndexSchema
+                          .builder()
+                          .withDimensionsSpec(dimensionsSpec)
+                          .withMetrics(new CountAggregatorFactory("cnt"))
+                          .build()
+                  )
+                  .setMaxRowCount(1_000)
+                  .build();
+            }
+        )
+    );
   }
 
-  @Parameterized.Parameters(name = "{index}: {0}")
+  @Parameterized.Parameters(name = "{index}: {0}, sortByDim: {1}")
   public static Collection<?> constructorFeeder()
   {
-    return IncrementalIndexCreator.getAppendableIndexTypes();
+    return IncrementalIndexCreator.indexTypeCartesianProduct(
+        ImmutableList.of(true, false) // sortByDim
+    );
   }
 
   @Test
@@ -155,6 +194,7 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
       final Sequence<ResultRow> rows = GroupByQueryEngine.process(
           query,
           adapter,
+          new IncrementalIndexTimeBoundaryInspector(index),
           cursorHolder,
           buildSpec,
           processingBuffer.get(),
@@ -231,6 +271,7 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
       final Sequence<ResultRow> rows = GroupByQueryEngine.process(
           query,
           adapter,
+          new IncrementalIndexTimeBoundaryInspector(index),
           cursorHolder,
           buildSpec,
           processingBuffer.get(),
@@ -257,6 +298,8 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
   @Test
   public void testResetSanity() throws IOException
   {
+    // Test is only valid when sortByDim = false, due to usage of Granularities.NONE.
+    Assume.assumeFalse(sortByDim);
 
     IncrementalIndex index = indexCreator.createIndex();
     DateTime t = DateTimes.nowUtc();
@@ -280,13 +323,13 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
     IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index);
 
     for (boolean descending : Arrays.asList(false, true)) {
-      final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
-                                                       .setFilter(new SelectorFilter("sally", "bo"))
-                                                       .setInterval(interval)
-                                                       .setPreferredOrdering(
-                                                           descending ? Cursors.descendingTimeOrder() : null
-                                                       )
-                                                       .build();
+      final CursorBuildSpec buildSpec = CursorBuildSpec
+          .builder()
+          .setFilter(new SelectorFilter("sally", "bo"))
+          .setInterval(interval)
+          .setPreferredOrdering(descending ? Cursors.descendingTimeOrder() : Cursors.ascendingTimeOrder())
+          .build();
+
       try (final CursorHolder cursorHolder = adapter.makeCursorHolder(buildSpec)) {
         Cursor cursor = cursorHolder.asCursor();
         DimensionSelector dimSelector;
@@ -348,6 +391,7 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
                   .aggregators(new LongSumAggregatorFactory("cnt", "cnt"))
                   .build(),
               new IncrementalIndexStorageAdapter(index),
+              new IncrementalIndexTimeBoundaryInspector(index),
               null
           )
           .toList();
@@ -400,6 +444,7 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
       final Sequence<ResultRow> rows = GroupByQueryEngine.process(
           query,
           adapter,
+          new IncrementalIndexTimeBoundaryInspector(index),
           cursorHolder,
           buildSpec,
           processingBuffer.get(),
@@ -519,14 +564,19 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
     final IncrementalIndex index = indexCreator.createIndex();
     final long timestamp = System.currentTimeMillis();
 
+    final List<InputRow> rows = ImmutableList.of(
+        new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v00")),
+        new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v01")),
+        new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v1")),
+        new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2")),
+        new MapBasedInputRow(timestamp, Collections.singletonList("billy2"), ImmutableMap.of("billy2", "v3")),
+        new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v3")),
+        new MapBasedInputRow(timestamp, Collections.singletonList("billy3"), ImmutableMap.of("billy3", ""))
+    );
+
+    // Add first two rows.
     for (int i = 0; i < 2; i++) {
-      index.add(
-          new MapBasedInputRow(
-              timestamp,
-              Collections.singletonList("billy"),
-              ImmutableMap.of("billy", "v0" + i)
-          )
-      );
+      index.add(rows.get(i));
     }
 
     final StorageAdapter sa = new IncrementalIndexStorageAdapter(index);
@@ -544,7 +594,7 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
 
       //index gets more rows at this point, while other thread is iterating over the cursor
       try {
-        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v1")));
+        index.add(rows.get(2));
       }
       catch (Exception ex) {
         throw new RuntimeException(ex);
@@ -555,12 +605,8 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
           .makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
       //index gets more rows at this point, while other thread is iterating over the cursor
       try {
-        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2")));
-        index.add(new MapBasedInputRow(
-            timestamp,
-            Collections.singletonList("billy2"),
-            ImmutableMap.of("billy2", "v3")
-        ));
+        index.add(rows.get(3));
+        index.add(rows.get(4));
       }
       catch (Exception ex) {
         throw new RuntimeException(ex);
@@ -575,8 +621,8 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
           .makeDimensionSelector(new DefaultDimensionSpec("billy2", "billy2"));
       //index gets more rows at this point, while other thread is iterating over the cursor
       try {
-        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v3")));
-        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy3"), ImmutableMap.of("billy3", "")));
+        index.add(rows.get(5));
+        index.add(rows.get(6));
       }
       catch (Exception ex) {
         throw new RuntimeException(ex);
diff --git a/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
index e323b259913..b26da1ceba7 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
@@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.math.expr.ExprMacroTable;
@@ -127,24 +126,6 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
     );
   }
 
-  @Test
-  public void test_getMinTime_factToCountry()
-  {
-    Assert.assertEquals(
-        DateTimes.of("2015-09-12T00:46:58.771Z"),
-        makeFactToCountrySegment().getMinTime()
-    );
-  }
-
-  @Test
-  public void test_getMaxTime_factToCountry()
-  {
-    Assert.assertEquals(
-        DateTimes.of("2015-09-12T05:21:00.059Z"),
-        makeFactToCountrySegment().getMaxTime()
-    );
-  }
-
   @Test
   public void test_getMinValue_factToCountryFactColumn()
   {
@@ -268,15 +249,6 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
     );
   }
 
-  @Test
-  public void test_getMaxIngestedEventTime_factToCountry()
-  {
-    Assert.assertEquals(
-        DateTimes.of("2015-09-12T05:21:00.059Z"),
-        makeFactToCountrySegment().getMaxIngestedEventTime()
-    );
-  }
-
   @Test
   public void test_getNumRows_factToCountry()
   {
diff --git a/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentTest.java b/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentTest.java
index 581c9a1c705..9f9905e47d4 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentTest.java
@@ -20,13 +20,16 @@
 package org.apache.druid.segment.join;
 
 import com.google.common.collect.ImmutableList;
+import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.segment.MaxIngestedEventTimeInspector;
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.QueryableIndexSegment;
 import org.apache.druid.segment.ReferenceCountingSegment;
 import org.apache.druid.segment.SegmentReference;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.join.table.IndexedTableJoinable;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.apache.druid.timeline.SegmentId;
@@ -39,6 +42,7 @@ import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.Closeable;
 import java.io.IOException;
@@ -164,6 +168,13 @@ public class HashJoinSegmentTest extends InitializedNullHandlingTest
         return referencedSegment.asStorageAdapter();
       }
 
+      @Nullable
+      @Override
+      public <T> T as(@Nonnull Class<T> clazz)
+      {
+        return referencedSegment.as(clazz);
+      }
+
       @Override
       public void close()
       {
@@ -308,4 +319,22 @@ public class HashJoinSegmentTest extends InitializedNullHandlingTest
     Assert.assertEquals(1, indexedTableJoinableReferenceCloseCount);
     Assert.assertEquals(0, allReferencesCloseCount);
   }
+
+
+  @Test
+  public void testGetMinTime()
+  {
+    final TimeBoundaryInspector timeBoundaryInspector = hashJoinSegment.as(TimeBoundaryInspector.class);
+    Assert.assertNotNull("non-null inspector", timeBoundaryInspector);
+    Assert.assertEquals("minTime", DateTimes.of("2015-09-12T00:46:58.771Z"), timeBoundaryInspector.getMinTime());
+    Assert.assertEquals("maxTime", DateTimes.of("2015-09-12T05:21:00.059Z"), timeBoundaryInspector.getMaxTime());
+    Assert.assertFalse("exact", timeBoundaryInspector.isMinMaxExact());
+  }
+
+  @Test
+  public void testGetMaxIngestedEventTime()
+  {
+    final MaxIngestedEventTimeInspector inspector = referencedSegment.as(MaxIngestedEventTimeInspector.class);
+    Assert.assertNull(inspector);
+  }
 }
diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java
index 2d929f079bf..73daadc7dd1 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java
@@ -326,7 +326,7 @@ public class BroadcastSegmentIndexedTableTest extends InitializedNullHandlingTes
       closer.register(theColumn);
       final BaseObjectColumnValueSelector<?> selector = theColumn.makeColumnValueSelector(offset);
 
-      ColumnSelectorFactory tableFactory = broadcastTable.makeColumnSelectorFactory(offset, false, closer);
+      ColumnSelectorFactory tableFactory = broadcastTable.makeColumnSelectorFactory(offset, closer);
       final BaseObjectColumnValueSelector<?> tableSelector = tableFactory.makeColumnValueSelector(columnName);
 
       // compare with base segment selector to make sure tables selector can read correct values
diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/IndexedTableJoinableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/IndexedTableJoinableTest.java
index 09ddec72b6b..b08249f2a4d 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/table/IndexedTableJoinableTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/table/IndexedTableJoinableTest.java
@@ -181,7 +181,6 @@ public class IndexedTableJoinableTest
         dummyColumnSelectorFactory,
         condition,
         false,
-        false,
         Closer.create()
     );
 
diff --git a/processing/src/test/java/org/apache/druid/timeline/partition/DimensionRangeShardSpecTest.java b/processing/src/test/java/org/apache/druid/timeline/partition/DimensionRangeShardSpecTest.java
index fa49a48bcfa..ade40e6ae38 100644
--- a/processing/src/test/java/org/apache/druid/timeline/partition/DimensionRangeShardSpecTest.java
+++ b/processing/src/test/java/org/apache/druid/timeline/partition/DimensionRangeShardSpecTest.java
@@ -406,6 +406,29 @@ public class DimensionRangeShardSpecTest
         // EffectiveDomain[:3].size > 1 -> ACCEPT
     );
     assertTrue(shard.possibleInDomain(domain));
+
+    // {Earth} * {France} * {Paris}
+    // Tests that we accept the ending point of the range
+    populateDomain(
+        domain,
+        getRangeSet(Range.singleton("Earth")),
+        getRangeSet(Range.singleton("France")),
+        getRangeSet(Range.singleton("Paris"))
+    );
+    assertTrue(shard.possibleInDomain(domain));
+
+    // {Earth} * {USA} * {New York}
+    // Tests that we accept the ending point of the range
+    // This is important because shard specs generated by MSQ with "boosting" do not include the __boost column,
+    // which means that a data point matching the end of a range may be in this segment or may be in the next
+    // segment (so we do need to check both).
+    populateDomain(
+        domain,
+        getRangeSet(Range.singleton("Earth")),
+        getRangeSet(Range.singleton("USA")),
+        getRangeSet(Range.singleton("New York"))
+    );
+    assertTrue(shard.possibleInDomain(domain));
   }
 
   @Test
diff --git a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java
index 1e717f5d17b..bda88401881 100644
--- a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java
+++ b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java
@@ -31,15 +31,18 @@ import com.google.common.base.Strings;
 import com.google.common.collect.Multiset;
 import com.google.common.collect.TreeMultiset;
 import org.apache.druid.common.utils.IdUtils;
+import org.apache.druid.data.input.impl.DimensionSchema;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.InputRowParser;
 import org.apache.druid.data.input.impl.ParseSpec;
 import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.indexing.granularity.GranularitySpec;
 import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
 import org.apache.druid.segment.transform.TransformSpec;
@@ -212,16 +215,51 @@ public class DataSchema
 
     fields.computeIfAbsent(ColumnHolder.TIME_COLUMN_NAME, k -> TreeMultiset.create()).add(
         StringUtils.format(
-            "primary timestamp (%s cannot appear as a dimension or metric)",
+            "primary timestamp (%s cannot appear elsewhere except as long-typed dimension)",
             ColumnHolder.TIME_COLUMN_NAME
         )
     );
 
     if (dimensionsSpec != null) {
+      boolean sawTimeDimension = false;
+
       for (int i = 0; i < dimensionsSpec.getDimensions().size(); i++) {
-        final String field = dimensionsSpec.getDimensions().get(i).getName();
+        final DimensionSchema dimSchema = dimensionsSpec.getDimensions().get(i);
+        final String field = dimSchema.getName();
         if (Strings.isNullOrEmpty(field)) {
-          throw new IAE("Encountered dimension with null or empty name at position %d", i);
+          throw DruidException
+              .forPersona(DruidException.Persona.USER)
+              .ofCategory(DruidException.Category.INVALID_INPUT)
+              .build("Encountered dimension with null or empty name at position[%d]", i);
+        }
+
+        if (ColumnHolder.TIME_COLUMN_NAME.equals(field)) {
+          if (i > 0 && dimensionsSpec.isForceSegmentSortByTime()) {
+            throw DruidException
+                .forPersona(DruidException.Persona.USER)
+                .ofCategory(DruidException.Category.INVALID_INPUT)
+                .build(
+                    "Encountered dimension[%s] at position[%d]. This is only supported when the dimensionsSpec "
+                    + "parameter[%s] is set to[false]. %s",
+                    field,
+                    i,
+                    DimensionsSpec.PARAMETER_FORCE_TIME_SORT,
+                    DimensionsSpec.WARNING_NON_TIME_SORT_ORDER
+                );
+          } else if (!dimSchema.getColumnType().is(ValueType.LONG)) {
+            throw DruidException
+                .forPersona(DruidException.Persona.USER)
+                .ofCategory(DruidException.Category.INVALID_INPUT)
+                .build(
+                    "Encountered dimension[%s] with incorrect type[%s]. Type must be 'long'.",
+                    field,
+                    dimSchema.getColumnType()
+                );
+          } else if (!sawTimeDimension) {
+            // Skip adding __time to "fields" (once) if it's listed as a dimension, so it doesn't show up as an error.
+            sawTimeDimension = true;
+            continue;
+          }
         }
 
         fields.computeIfAbsent(field, k -> TreeMultiset.create()).add("dimensions list");
@@ -266,7 +304,9 @@ public class DataSchema
     if (errors.isEmpty()) {
       return fields.keySet();
     } else {
-      throw new IAE("Cannot specify a column more than once: %s", String.join("; ", errors));
+      throw DruidException.forPersona(DruidException.Persona.USER)
+                          .ofCategory(DruidException.Category.INVALID_INPUT)
+                          .build("Cannot specify a column more than once: %s", String.join("; ", errors));
     }
   }
 
diff --git a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java
index 72cfe4de261..557198f1424 100644
--- a/server/src/main/java/org/apache/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java
+++ b/server/src/main/java/org/apache/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java
@@ -28,7 +28,6 @@ import org.apache.druid.client.cache.CacheConfig;
 import org.apache.druid.client.cache.CachePopulatorStats;
 import org.apache.druid.client.cache.ForegroundCachePopulator;
 import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.guava.FunctionalIterable;
 import org.apache.druid.java.util.emitter.EmittingLogger;
@@ -57,7 +56,7 @@ import org.apache.druid.query.planning.DataSourceAnalysis;
 import org.apache.druid.query.spec.SpecificSegmentQueryRunner;
 import org.apache.druid.query.spec.SpecificSegmentSpec;
 import org.apache.druid.segment.SegmentReference;
-import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.realtime.FireHydrant;
 import org.apache.druid.segment.realtime.sink.Sink;
 import org.apache.druid.segment.realtime.sink.SinkSegmentReference;
@@ -245,15 +244,21 @@ public class SinkQuerySegmentWalker implements QuerySegmentWalker
                     // 1) Only use caching if data is immutable
                     // 2) Hydrants are not the same between replicas, make sure cache is local
                     if (segmentReference.isImmutable() && cache.isLocal()) {
-                      StorageAdapter storageAdapter = segmentReference.getSegment().asStorageAdapter();
-                      long segmentMinTime = storageAdapter.getMinTime().getMillis();
-                      long segmentMaxTime = storageAdapter.getMaxTime().getMillis();
-                      Interval actualDataInterval = Intervals.utc(segmentMinTime, segmentMaxTime + 1);
+                      final SegmentReference segment = segmentReference.getSegment();
+                      final TimeBoundaryInspector timeBoundaryInspector = segment.as(TimeBoundaryInspector.class);
+                      final Interval cacheKeyInterval;
+
+                      if (timeBoundaryInspector != null) {
+                        cacheKeyInterval = timeBoundaryInspector.getMinMaxInterval();
+                      } else {
+                        cacheKeyInterval = segment.getDataInterval();
+                      }
+
                       runner = new CachingQueryRunner<>(
                           makeHydrantCacheIdentifier(sinkSegmentId, segmentReference.getHydrantNumber()),
                           cacheKeyPrefix,
                           descriptor,
-                          actualDataInterval,
+                          cacheKeyInterval,
                           objectMapper,
                           cache,
                           toolChest,
diff --git a/server/src/main/java/org/apache/druid/segment/realtime/sink/Sink.java b/server/src/main/java/org/apache/druid/segment/realtime/sink/Sink.java
index 44d0db3cb7f..8d4fc0dd7f8 100644
--- a/server/src/main/java/org/apache/druid/segment/realtime/sink/Sink.java
+++ b/server/src/main/java/org/apache/druid/segment/realtime/sink/Sink.java
@@ -349,7 +349,7 @@ public class Sink implements Iterable<FireHydrant>, Overshadowable<Sink>
           Map<String, ColumnFormat> oldFormat = null;
           newCount = lastHydrant.getCount() + 1;
 
-          boolean customDimensions = !indexSchema.getDimensionsSpec().hasCustomDimensions();
+          boolean variableDimensions = !indexSchema.getDimensionsSpec().hasFixedDimensions();
 
           if (lastHydrant.hasSwapped()) {
             oldFormat = new HashMap<>();
@@ -357,7 +357,7 @@ public class Sink implements Iterable<FireHydrant>, Overshadowable<Sink>
             try {
               QueryableIndex oldIndex = segment.asQueryableIndex();
               overwriteIndexDimensions(new QueryableIndexStorageAdapter(oldIndex));
-              if (customDimensions) {
+              if (variableDimensions) {
                 for (String dim : oldIndex.getAvailableDimensions()) {
                   dimOrder.add(dim);
                   oldFormat.put(dim, oldIndex.getColumnHolder(dim).getColumnFormat());
@@ -370,12 +370,12 @@ public class Sink implements Iterable<FireHydrant>, Overshadowable<Sink>
           } else {
             IncrementalIndex oldIndex = lastHydrant.getIndex();
             overwriteIndexDimensions(new IncrementalIndexStorageAdapter(oldIndex));
-            if (customDimensions) {
+            if (variableDimensions) {
               dimOrder.addAll(oldIndex.getDimensionOrder());
               oldFormat = oldIndex.getColumnFormats();
             }
           }
-          if (customDimensions) {
+          if (variableDimensions) {
             newIndex.loadDimensionIterable(dimOrder, oldFormat);
           }
         }
diff --git a/server/src/main/java/org/apache/druid/server/coordination/ServerManager.java b/server/src/main/java/org/apache/druid/server/coordination/ServerManager.java
index b479737c994..f3264dad44c 100644
--- a/server/src/main/java/org/apache/druid/server/coordination/ServerManager.java
+++ b/server/src/main/java/org/apache/druid/server/coordination/ServerManager.java
@@ -28,7 +28,6 @@ import org.apache.druid.client.cache.CacheConfig;
 import org.apache.druid.client.cache.CachePopulator;
 import org.apache.druid.guice.annotations.Smile;
 import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.guava.FunctionalIterable;
 import org.apache.druid.java.util.emitter.EmittingLogger;
@@ -60,6 +59,7 @@ import org.apache.druid.query.spec.SpecificSegmentSpec;
 import org.apache.druid.segment.ReferenceCountingSegment;
 import org.apache.druid.segment.SegmentReference;
 import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.TimeBoundaryInspector;
 import org.apache.druid.segment.join.JoinableFactoryWrapper;
 import org.apache.druid.server.ResourceIdPopulatingQueryRunner;
 import org.apache.druid.server.SegmentManager;
@@ -307,14 +307,14 @@ public class ServerManager implements QuerySegmentWalker
         queryMetrics -> queryMetrics.segment(segmentIdString)
     );
 
-    long segmentMaxTime = storageAdapter.getMaxTime().getMillis();
-    long segmentMinTime = storageAdapter.getMinTime().getMillis();
-    Interval actualDataInterval = Intervals.utc(segmentMinTime, segmentMaxTime + 1);
+    final TimeBoundaryInspector timeBoundaryInspector = segment.as(TimeBoundaryInspector.class);
+    final Interval cacheKeyInterval =
+        timeBoundaryInspector != null ? timeBoundaryInspector.getMinMaxInterval() : segmentInterval;
     CachingQueryRunner<T> cachingQueryRunner = new CachingQueryRunner<>(
         segmentIdString,
         cacheKeyPrefix,
         segmentDescriptor,
-        actualDataInterval,
+        cacheKeyInterval,
         objectMapper,
         cache,
         toolChest,
diff --git a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
index 78294fca0c4..32b2a3830fd 100644
--- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
+++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
@@ -30,8 +30,11 @@ import org.apache.druid.common.utils.IdUtilsTest;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.JSONParseSpec;
+import org.apache.druid.data.input.impl.LongDimensionSchema;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
 import org.apache.druid.data.input.impl.StringInputRowParser;
 import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.error.DruidExceptionMatcher;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
@@ -242,7 +245,7 @@ public class DataSchemaTest extends InitializedNullHandlingTest
         jsonMapper
     );
 
-    expectedException.expect(IllegalArgumentException.class);
+    expectedException.expect(DruidException.class);
     expectedException.expectMessage(
         "Cannot specify a column more than once: [metric1] seen in dimensions list, metricsSpec list"
     );
@@ -251,20 +254,18 @@ public class DataSchemaTest extends InitializedNullHandlingTest
   }
 
   @Test
-  public void testOverlapTimeAndDim()
+  public void testOverlapTimeAndDimPositionZero()
   {
-    expectedException.expect(IllegalArgumentException.class);
-    expectedException.expectMessage(
-        "Cannot specify a column more than once: [__time] seen in dimensions list, "
-        + "primary timestamp (__time cannot appear as a dimension or metric)"
-    );
-
     DataSchema schema = new DataSchema(
         IdUtilsTest.VALID_ID_CHARS,
         new TimestampSpec("time", "auto", null),
         DimensionsSpec.builder()
                       .setDimensions(
-                          DimensionsSpec.getDefaultSchemas(ImmutableList.of("__time", "dimA", "dimB", "metric1"))
+                          ImmutableList.of(
+                              new LongDimensionSchema("__time"),
+                              new StringDimensionSchema("dimA"),
+                              new StringDimensionSchema("dimB")
+                          )
                       )
                       .setDimensionExclusions(ImmutableList.of("dimC"))
                       .build(),
@@ -274,6 +275,103 @@ public class DataSchemaTest extends InitializedNullHandlingTest
         null,
         jsonMapper
     );
+
+    Assert.assertEquals(
+        ImmutableList.of("__time", "dimA", "dimB"),
+        schema.getDimensionsSpec().getDimensionNames()
+    );
+
+    Assert.assertTrue(schema.getDimensionsSpec().isForceSegmentSortByTime());
+  }
+
+  @Test
+  public void testOverlapTimeAndDimPositionZeroWrongType()
+  {
+    expectedException.expect(DruidException.class);
+    expectedException.expectMessage("Encountered dimension[__time] with incorrect type[STRING]. Type must be 'long'.");
+
+    DataSchema schema = new DataSchema(
+        IdUtilsTest.VALID_ID_CHARS,
+        new TimestampSpec("time", "auto", null),
+        DimensionsSpec.builder()
+                      .setDimensions(
+                          ImmutableList.of(
+                              new StringDimensionSchema("__time"),
+                              new StringDimensionSchema("dimA"),
+                              new StringDimensionSchema("dimB")
+                          )
+                      )
+                      .setDimensionExclusions(ImmutableList.of("dimC"))
+                      .build(),
+        null,
+        new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))),
+        null,
+        null,
+        jsonMapper
+    );
+  }
+
+  @Test
+  public void testOverlapTimeAndDimPositionOne()
+  {
+    expectedException.expect(DruidException.class);
+    expectedException.expectMessage(
+        "Encountered dimension[__time] at position[1]. This is only supported when the dimensionsSpec "
+        + "parameter[forceSegmentSortByTime] is set to[false]. "
+        + DimensionsSpec.WARNING_NON_TIME_SORT_ORDER
+    );
+
+    DataSchema schema = new DataSchema(
+        IdUtilsTest.VALID_ID_CHARS,
+        new TimestampSpec("time", "auto", null),
+        DimensionsSpec.builder()
+                      .setDimensions(
+                          ImmutableList.of(
+                              new StringDimensionSchema("dimA"),
+                              new LongDimensionSchema("__time"),
+                              new StringDimensionSchema("dimB")
+                          )
+                      )
+                      .setDimensionExclusions(ImmutableList.of("dimC"))
+                      .build(),
+        null,
+        new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))),
+        null,
+        null,
+        jsonMapper
+    );
+  }
+
+  @Test
+  public void testOverlapTimeAndDimPositionOne_withExplicitSortOrder()
+  {
+    DataSchema schema = new DataSchema(
+        IdUtilsTest.VALID_ID_CHARS,
+        new TimestampSpec("time", "auto", null),
+        DimensionsSpec.builder()
+                      .setDimensions(
+                          ImmutableList.of(
+                              new StringDimensionSchema("dimA"),
+                              new LongDimensionSchema("__time"),
+                              new StringDimensionSchema("dimB")
+                          )
+                      )
+                      .setDimensionExclusions(ImmutableList.of("dimC"))
+                      .setForceSegmentSortByTime(false)
+                      .build(),
+        null,
+        new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))),
+        null,
+        null,
+        jsonMapper
+    );
+
+    Assert.assertEquals(
+        ImmutableList.of("dimA", "__time", "dimB"),
+        schema.getDimensionsSpec().getDimensionNames()
+    );
+
+    Assert.assertFalse(schema.getDimensionsSpec().isForceSegmentSortByTime());
   }
 
   @Test
@@ -313,11 +411,8 @@ public class DataSchemaTest extends InitializedNullHandlingTest
         jsonMapper
     );
 
-    expectedException.expect(IllegalArgumentException.class);
-    expectedException.expectMessage(
-        "Cannot specify a column more than once: [__time] seen in dimensions list, primary timestamp "
-        + "(__time cannot appear as a dimension or metric)"
-    );
+    expectedException.expect(DruidException.class);
+    expectedException.expectMessage("Encountered dimension[__time] with incorrect type[STRING]. Type must be 'long'.");
 
     schema.getParser();
   }
@@ -341,7 +436,7 @@ public class DataSchemaTest extends InitializedNullHandlingTest
         ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT
     );
 
-    expectedException.expect(IllegalArgumentException.class);
+    expectedException.expect(DruidException.class);
     expectedException.expectMessage(
         "Cannot specify a column more than once: [metric1] seen in metricsSpec list (2 occurrences); "
         + "[metric3] seen in metricsSpec list (2 occurrences)"
diff --git a/server/src/test/java/org/apache/druid/server/TestSegmentUtils.java b/server/src/test/java/org/apache/druid/server/TestSegmentUtils.java
index 14eb3d44220..866381dd70c 100644
--- a/server/src/test/java/org/apache/druid/server/TestSegmentUtils.java
+++ b/server/src/test/java/org/apache/druid/server/TestSegmentUtils.java
@@ -27,9 +27,11 @@ import com.google.common.io.Files;
 import org.apache.druid.collections.bitmap.BitmapFactory;
 import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.Cursor;
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.DimensionHandler;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.Metadata;
@@ -46,7 +48,6 @@ import org.apache.druid.segment.loading.SegmentizerFactory;
 import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.SegmentId;
 import org.apache.druid.timeline.partition.NoneShardSpec;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
 
@@ -174,6 +175,12 @@ public class TestSegmentUtils
         throw new UnsupportedOperationException();
       }
 
+      @Override
+      public List<OrderBy> getOrdering()
+      {
+        return Cursors.ascendingTimeOrder();
+      }
+
       @Override
       public void close()
       {
@@ -264,19 +271,6 @@ public class TestSegmentUtils
           return cardinality;
         }
 
-        @Override
-        public DateTime getMinTime()
-        {
-          return interval.getStart();
-        }
-
-
-        @Override
-        public DateTime getMaxTime()
-        {
-          return interval.getEnd();
-        }
-
         @Override
         public Indexed<String> getAvailableDimensions()
         {
@@ -316,12 +310,6 @@ public class TestSegmentUtils
           return 0;
         }
 
-        @Override
-        public DateTime getMaxIngestedEventTime()
-        {
-          return null;
-        }
-
         @Override
         public Metadata getMetadata()
         {
diff --git a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java
index 0ca08523fb2..f17d5129143 100644
--- a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java
+++ b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java
@@ -103,6 +103,7 @@ public class DumpSegmentTest extends InitializedNullHandlingTest
     QueryRunner runner = Mockito.mock(QueryRunner.class);
     QueryRunner mergeRunner = Mockito.mock(QueryRunner.class);
     Query query = Mockito.mock(Query.class);
+    QueryableIndex index = Mockito.mock(QueryableIndex.class);
     Sequence expected = Sequences.simple(Collections.singletonList(123));
     Mockito.when(query.withOverriddenContext(ArgumentMatchers.any())).thenReturn(query);
     Mockito.when(injector.getInstance(QueryRunnerFactoryConglomerate.class)).thenReturn(conglomerate);
@@ -111,7 +112,8 @@ public class DumpSegmentTest extends InitializedNullHandlingTest
     Mockito.when(factory.getToolchest().mergeResults(factory.mergeRunners(DirectQueryProcessingPool.INSTANCE, ImmutableList.of(runner)))).thenReturn(mergeRunner);
     Mockito.when(factory.getToolchest().mergeResults(factory.mergeRunners(DirectQueryProcessingPool.INSTANCE, ImmutableList.of(runner)), true)).thenReturn(mergeRunner);
     Mockito.when(mergeRunner.run(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(expected);
-    Sequence actual = DumpSegment.executeQuery(injector, null, query);
+    Mockito.when(index.getOrdering()).thenReturn(Collections.emptyList());
+    Sequence actual = DumpSegment.executeQuery(injector, index, query);
     Assert.assertSame(expected, actual);
   }
 
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java
index 21ad4c23663..79f9d79e849 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java
@@ -33,6 +33,7 @@ import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.Numbers;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.QueryContext;
@@ -112,6 +113,12 @@ public class PlannerContext
   public static final String CTX_SQL_REVERSE_LOOKUP = "sqlReverseLookup";
   public static final boolean DEFAULT_SQL_REVERSE_LOOKUP = true;
 
+  /**
+   * Context key for {@link PlannerContext#isUseGranularity()}.
+   */
+  public static final String CTX_SQL_USE_GRANULARITY = "sqlUseGranularity";
+  public static final boolean DEFAULT_SQL_USE_GRANULARITY = true;
+
   // DataContext keys
   public static final String DATA_CTX_AUTHENTICATION_RESULT = "authenticationResult";
 
@@ -127,6 +134,7 @@ public class PlannerContext
   private final boolean useBoundsAndSelectors;
   private final boolean pullUpLookup;
   private final boolean reverseLookup;
+  private final boolean useGranularity;
   private final CopyOnWriteArrayList<String> nativeQueryIds = new CopyOnWriteArrayList<>();
   private final PlannerHook hook;
   // bindings for dynamic parameters to bind during planning
@@ -156,6 +164,7 @@ public class PlannerContext
       final boolean useBoundsAndSelectors,
       final boolean pullUpLookup,
       final boolean reverseLookup,
+      final boolean useGranularity,
       final SqlEngine engine,
       final Map<String, Object> queryContext,
       final PlannerHook hook
@@ -172,6 +181,7 @@ public class PlannerContext
     this.useBoundsAndSelectors = useBoundsAndSelectors;
     this.pullUpLookup = pullUpLookup;
     this.reverseLookup = reverseLookup;
+    this.useGranularity = useGranularity;
     this.hook = hook == null ? NoOpPlannerHook.INSTANCE : hook;
 
     String sqlQueryId = (String) this.queryContext.get(QueryContexts.CTX_SQL_QUERY_ID);
@@ -196,6 +206,7 @@ public class PlannerContext
     final boolean useBoundsAndSelectors;
     final boolean pullUpLookup;
     final boolean reverseLookup;
+    final boolean useGranularity;
 
     final Object stringifyParam = queryContext.get(QueryContexts.CTX_SQL_STRINGIFY_ARRAYS);
     final Object tsParam = queryContext.get(CTX_SQL_CURRENT_TIMESTAMP);
@@ -203,6 +214,7 @@ public class PlannerContext
     final Object useBoundsAndSelectorsParam = queryContext.get(CTX_SQL_USE_BOUNDS_AND_SELECTORS);
     final Object pullUpLookupParam = queryContext.get(CTX_SQL_PULL_UP_LOOKUP);
     final Object reverseLookupParam = queryContext.get(CTX_SQL_REVERSE_LOOKUP);
+    final Object useGranularityParam = queryContext.get(CTX_SQL_USE_GRANULARITY);
 
     if (tsParam != null) {
       utcNow = new DateTime(tsParam, DateTimeZone.UTC);
@@ -240,6 +252,12 @@ public class PlannerContext
       reverseLookup = DEFAULT_SQL_REVERSE_LOOKUP;
     }
 
+    if (useGranularityParam != null) {
+      useGranularity = Numbers.parseBoolean(useGranularityParam);
+    } else {
+      useGranularity = DEFAULT_SQL_USE_GRANULARITY;
+    }
+
     return new PlannerContext(
         plannerToolbox,
         sql,
@@ -249,6 +267,7 @@ public class PlannerContext
         useBoundsAndSelectors,
         pullUpLookup,
         reverseLookup,
+        useGranularity,
         engine,
         queryContext,
         hook
@@ -425,6 +444,16 @@ public class PlannerContext
     return reverseLookup;
   }
 
+  /**
+   * Whether we should use granularities other than {@link Granularities#ALL} when planning queries. This is provided
+   * mainly so it can be set to false when running SQL queries on non-time-ordered tables, which do not support
+   * any other granularities.
+   */
+  public boolean isUseGranularity()
+  {
+    return useGranularity;
+  }
+
   public List<TypedValue> getParameters()
   {
     return parameters;
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
index dc127fa7f86..51cf9e923ed 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
@@ -1144,6 +1144,11 @@ public class DruidQuery
         // Timeseries only applies if the single dimension is granular __time.
         return null;
       }
+
+      if (!plannerContext.isUseGranularity() && !Granularities.ALL.equals(queryGranularity)) {
+        return null;
+      }
+
       theContext.put(
           TimeseriesQuery.CTX_TIMESTAMP_RESULT_FIELD,
           Iterables.getOnlyElement(grouping.getDimensions()).toDimensionSpec().getOutputName()
@@ -1406,7 +1411,7 @@ public class DruidQuery
     // the query generated by "explain plan for select ..." doesn't match to the native query ACTUALLY being executed,
     // the granularity and dimensions are slightly different.
     // now, part of the query plan logic is handled in GroupByStrategyV2.
-    if (!grouping.getDimensions().isEmpty()) {
+    if (!grouping.getDimensions().isEmpty() && plannerContext.isUseGranularity()) {
       for (DimensionExpression dimensionExpression : grouping.getDimensions()) {
         Granularity granularity = Expressions.toQueryGranularity(
             dimensionExpression.getDruidExpression(),
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
index 546776575f2..04113067cb1 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
@@ -3989,10 +3989,8 @@ public class CalciteJoinQueryTest extends BaseCalciteQueryTest
   @ParameterizedTest(name = "{0}")
   public void testTwoSemiJoinsSimultaneously(Map<String, Object> queryContext)
   {
-    // Fully removing the join allows this query to vectorize.
-    if (!isRewriteJoinToFilter(queryContext)) {
-      cannotVectorize();
-    }
+    // Cannot vectorize timeBoundary with maxTime (the engine will request descending order, which cannot vectorize).
+    cannotVectorize();
 
     Map<String, Object> updatedQueryContext = new HashMap<>(queryContext);
     updatedQueryContext.put(QueryContexts.TIME_BOUNDARY_PLANNING_KEY, true);
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 556b437e758..58d29a738c0 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -9726,6 +9726,52 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
     );
   }
 
+  @Test
+  public void testTimeseriesDontUseGranularity()
+  {
+    // When sqlUseGranularity: false, this query plans as a groupBy rather than a timeseries.
+    final Map<String, Object> context = QueryContexts.override(
+        QUERY_CONTEXT_DEFAULT,
+        PlannerContext.CTX_SQL_USE_GRANULARITY,
+        false
+    );
+
+    testQuery(
+        "SELECT SUM(cnt), gran FROM (\n"
+        + "  SELECT floor(__time TO month) AS gran,\n"
+        + "  cnt FROM druid.foo\n"
+        + ") AS x\n"
+        + "GROUP BY gran\n"
+        + "ORDER BY gran",
+        context,
+        ImmutableList.of(
+            GroupByQuery
+                .builder()
+                .setDataSource(CalciteTests.DATASOURCE1)
+                .setInterval(querySegmentSpec(Filtration.eternity()))
+                .setGranularity(Granularities.ALL)
+                .setVirtualColumns(
+                    expressionVirtualColumn("v0", "timestamp_floor(\"__time\",'P1M',null,'UTC')", ColumnType.LONG))
+                .setDimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.LONG))
+                .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
+                .setLimitSpec(
+                    queryFramework().engine().featureAvailable(EngineFeature.GROUPBY_IMPLICITLY_SORTS)
+                    ? NoopLimitSpec.instance()
+                    : new DefaultLimitSpec(
+                        ImmutableList.of(new OrderByColumnSpec("d0", Direction.ASCENDING, StringComparators.NUMERIC)),
+                        Integer.MAX_VALUE
+                    )
+                )
+                .setContext(context)
+                .build()
+        ),
+        ImmutableList.of(
+            new Object[]{3L, timestamp("2000-01-01")},
+            new Object[]{3L, timestamp("2001-01-01")}
+        )
+    );
+  }
+
   @Test
   public void testFilteredTimeAggregators()
   {
@@ -11180,6 +11226,73 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
     );
   }
 
+  @Test
+  public void testGroupByTimeAndOtherDimensionDontUseGranularity()
+  {
+    // When sqlUseGranularity: false, this query doesn't use a timestamp result context.
+    final Map<String, Object> context = QueryContexts.override(
+        QUERY_CONTEXT_DEFAULT,
+        PlannerContext.CTX_SQL_USE_GRANULARITY,
+        false
+    );
+
+    testQuery(
+        "SELECT dim2, gran, SUM(cnt)\n"
+        + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, dim2, cnt FROM druid.foo) AS x\n"
+        + "GROUP BY dim2, gran\n"
+        + "ORDER BY dim2, gran",
+        ImmutableList.of(
+            GroupByQuery.builder()
+                        .setDataSource(CalciteTests.DATASOURCE1)
+                        .setInterval(querySegmentSpec(Filtration.eternity()))
+                        .setGranularity(Granularities.ALL)
+                        .setVirtualColumns(
+                            expressionVirtualColumn(
+                                "v0",
+                                "timestamp_floor(\"__time\",'P1M',null,'UTC')",
+                                ColumnType.LONG
+                            )
+                        )
+                        .setDimensions(
+                            dimensions(
+                                new DefaultDimensionSpec("dim2", "d0"),
+                                new DefaultDimensionSpec("v0", "d1", ColumnType.LONG)
+                            )
+                        )
+                        .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
+                        .setLimitSpec(
+                            queryFramework().engine().featureAvailable(EngineFeature.GROUPBY_IMPLICITLY_SORTS)
+                            ? NoopLimitSpec.instance()
+                            : new DefaultLimitSpec(
+                                ImmutableList.of(
+                                    new OrderByColumnSpec("d0", Direction.ASCENDING, StringComparators.LEXICOGRAPHIC),
+                                    new OrderByColumnSpec("d1", Direction.ASCENDING, StringComparators.NUMERIC)
+                                ),
+                                Integer.MAX_VALUE
+                            )
+                        )
+                        .setContext(context)
+                        .build()
+        ),
+        NullHandling.replaceWithDefault() ?
+        ImmutableList.of(
+            new Object[]{"", timestamp("2000-01-01"), 2L},
+            new Object[]{"", timestamp("2001-01-01"), 1L},
+            new Object[]{"a", timestamp("2000-01-01"), 1L},
+            new Object[]{"a", timestamp("2001-01-01"), 1L},
+            new Object[]{"abc", timestamp("2001-01-01"), 1L}
+        ) :
+        ImmutableList.of(
+            new Object[]{null, timestamp("2000-01-01"), 1L},
+            new Object[]{null, timestamp("2001-01-01"), 1L},
+            new Object[]{"", timestamp("2000-01-01"), 1L},
+            new Object[]{"a", timestamp("2000-01-01"), 1L},
+            new Object[]{"a", timestamp("2001-01-01"), 1L},
+            new Object[]{"abc", timestamp("2001-01-01"), 1L}
+        )
+    );
+  }
+
   @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.IMPROVED_PLAN, separateDefaultModeTest = true)
   @Test
   public void testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim()
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java
index a6be8f7cf99..8fde5d166b6 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java
@@ -197,6 +197,9 @@ public class CalciteTimeBoundaryQueryTest extends BaseCalciteQueryTest
   @Test
   public void testMaxTimeQueryWithJoin()
   {
+    // Cannot vectorize timeBoundary with maxTime (the engine will request descending order, which cannot vectorize).
+    cannotVectorize();
+
     HashMap<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);
     context.put(QueryContexts.TIME_BOUNDARY_PLANNING_KEY, true);
 

From 60d43179689f613a8df343c5741153c6e8688fe5 Mon Sep 17 00:00:00 2001
From: Hugh Evans <101668034+hevansDev@users.noreply.github.com>
Date: Fri, 23 Aug 2024 16:44:19 +0100
Subject: [PATCH 51/99] Linked back to query granularity docs (#16883)

* Linked back to query granularity docs

* Update ingestion-spec.md

clairfy about query granularities in the spec.

* Update docs/design/storage.md

Co-authored-by: Charles Smith <techdocsmith@gmail.com>

* Update docs/ingestion/ingestion-spec.md

Co-authored-by: Charles Smith <techdocsmith@gmail.com>

* Update docs/querying/granularities.md

Co-authored-by: Charles Smith <techdocsmith@gmail.com>

* Apply suggestions from code review

---------

Co-authored-by: Charles Smith <techdocsmith@gmail.com>
---
 docs/design/storage.md           |  2 +-
 docs/ingestion/ingestion-spec.md | 14 +++++++-------
 docs/querying/granularities.md   |  3 ++-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/docs/design/storage.md b/docs/design/storage.md
index 73e0b85fa9a..85914e7ada3 100644
--- a/docs/design/storage.md
+++ b/docs/design/storage.md
@@ -67,7 +67,7 @@ On the Coordinator / Historical side:
 Segments all have a four-part identifier with the following components:
 
 - Datasource name.
-- Time interval (for the time chunk containing the segment; this corresponds to the `segmentGranularity` specified at ingestion time).
+- Time interval for the time chunk containing the segment; this corresponds to the `segmentGranularity` specified at ingestion time. Uses the same format as [query granularity](../querying/granularities.md).
 - Version number (generally an ISO8601 timestamp corresponding to when the segment set was first started).
 - Partition number (an integer, unique within a datasource+interval+version; may not necessarily be contiguous).
 
diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md
index 37f0bf0a1fd..f1e974a9cc2 100644
--- a/docs/ingestion/ingestion-spec.md
+++ b/docs/ingestion/ingestion-spec.md
@@ -301,15 +301,15 @@ An example `metricsSpec` is:
 
 ### `granularitySpec`
 
-The `granularitySpec` is located in `dataSchema` → `granularitySpec` and is responsible for configuring
-the following operations:
+The `granularitySpec`, located in `dataSchema` → `granularitySpec`, specifies the following:
 
-1. Partitioning a datasource into [time chunks](../design/storage.md) (via `segmentGranularity`).
-2. Truncating the timestamp, if desired (via `queryGranularity`).
-3. Specifying which time chunks of segments should be created, for batch ingestion (via `intervals`).
-4. Specifying whether ingestion-time [rollup](./rollup.md) should be used or not (via `rollup`).
+1. `segmentGranularity` to partitioning a datasource into [time chunks](../design/storage.md).
+2. `queryGranularity` to optionally truncate the timestamp.
+3. `intervals` to define the time chunks of segments to create for batch ingestion.
+4.  `rollup` to enable ingestion-time [rollup](./rollup.md) or not.
 
 Other than `rollup`, these operations are all based on the [primary timestamp](./schema-model.md#primary-timestamp).
+Use the format from [Query granularities] to specify both `segmentGranualarity` and `queryGranularity`.
 
 An example `granularitySpec` is:
 
@@ -561,4 +561,4 @@ For example:
     }
   }
 }
-```
\ No newline at end of file
+```
diff --git a/docs/querying/granularities.md b/docs/querying/granularities.md
index c9895fa1ee0..6c204df1052 100644
--- a/docs/querying/granularities.md
+++ b/docs/querying/granularities.md
@@ -30,7 +30,8 @@ sidebar_label: "Granularities"
  [SQL documentation](sql-scalar.md#date-and-time-functions).
 :::
 
-Granularity determines how to bucket data across the time dimension, or how to aggregate data by hour, day, minute, etc.
+Granularity determines how to bucket data across the time dimension, or how to aggregate data by hour, day, minute, etc and defines how it is stored.
+The granularity formats here apply also to `segmentGranularity` and `queryGranularity` in the `granularitySpec` section of the the [ingestion spec](../ingestion/ingestion-spec.md#granularityspec).
 
 For example, use time granularities in [native queries](querying.md) to bucket results by time, and in the `dataSchema` \\ [`granularitySpec`](../ingestion/ingestion-spec.md#granularityspec) section of ingestion specifications to segment incoming data.
 

From 418da922281db3a131ac5ebefdedf8b774d67ac0 Mon Sep 17 00:00:00 2001
From: 317brian <53799971+317brian@users.noreply.github.com>
Date: Fri, 23 Aug 2024 11:59:29 -0700
Subject: [PATCH 52/99] docs: update query from deepstorage segment requirement
 (#16842)

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>
Co-authored-by: Rishabh Singh <6513075+findingrish@users.noreply.github.com>
Co-authored-by: Katya Macedo <38017980+ektravel@users.noreply.github.com>
---
 docs/configuration/index.md                   |  4 +++-
 docs/querying/query-from-deep-storage.md      | 18 +++++++++++-------
 docs/tutorials/tutorial-query-deep-storage.md |  2 +-
 website/.spelling                             |  1 +
 4 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 637ebeabc0e..9c83946eb82 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -596,7 +596,9 @@ need arises.
 |`druid.centralizedDatasourceSchema.enabled`|Boolean flag for enabling datasource schema building in the Coordinator, this should be specified in the common runtime properties.|false|No.|
 |`druid.indexer.fork.property.druid.centralizedDatasourceSchema.enabled`| This config should be set when CentralizedDatasourceSchema feature is enabled. This should be specified in the MiddleManager runtime properties.|false|No.|
 
-For, stale schema cleanup configs, refer to properties with the prefix `druid.coordinator.kill.segmentSchema` in [Metadata Management](#metadata-management).
+If you enable this feature, you can query datasources that are only stored in deep storage and are not loaded on a Historical. For more information, see [Query from deep storage](../querying/query-from-deep-storage.md).
+
+For stale schema cleanup configs, refer to properties with the prefix `druid.coordinator.kill.segmentSchema` in [Metadata Management](#metadata-management). 
 
 ### Ingestion security configuration
 
diff --git a/docs/querying/query-from-deep-storage.md b/docs/querying/query-from-deep-storage.md
index 1ce74818655..e90f2fb4b02 100644
--- a/docs/querying/query-from-deep-storage.md
+++ b/docs/querying/query-from-deep-storage.md
@@ -28,13 +28,20 @@ Druid can query segments that are only stored in deep storage. Running a query f
 
 Query from deep storage requires the Multi-stage query (MSQ) task engine. Load the extension for it if you don't already have it enabled before you begin. See [enable MSQ](../multi-stage-query/index.md#load-the-extension) for more information.
 
+To be queryable, your datasource must meet one of the following conditions:
+
+- At least one segment from the datasource is loaded onto a Historical service for Druid to plan the query. This segment can be any segment from the datasource. You can verify that a datasource has at least one segment on a Historical service if it's visible in the Druid console.
+- You have the centralized datasource schema feature enabled. For more information, see [Centralized datasource schema](../configuration/index.md#centralized-datasource-schema).
+
+If you use centralized data source schema, there's an additional step for any datasource created prior to enabling it to make the datasource queryable from deep storage. You need to load the segments from deep storage onto a Historical so that the schema can be backfilled in the metadata database. You can load some or all of the segments that are only in deep storage. If you don't load all the segments, any dimensions that are only in the segments you didn't load will not be in the queryable datasource schema and won't be queryable from deep storage. That is, only the dimensions that are present in the segment schema in metadata database are queryable. Once that process is complete, you can unload all the segments from the Historical and only keep the data in deep storage.
+
 ## Keep segments in deep storage only
 
-Any data you ingest into Druid is already stored in deep storage, so you don't need to perform any additional configuration from that perspective. However, to take advantage of the cost savings that querying from deep storage provides, make sure not all your segments get loaded onto Historical processes.
+Any data you ingest into Druid is already stored in deep storage, so you don't need to perform any additional configuration from that perspective. However, to take advantage of the cost savings that querying from deep storage provides, make sure not all your segments get loaded onto Historical processes. If you use centralized datasource schema, a datasource can be kept only in deep storage but remain queryable.
 
-To do this, configure [load rules](../operations/rule-configuration.md#load-rules) to manage the which segments are only in deep storage and which get loaded onto Historical processes.
+To manage which segments are kept only in deep storage and which get loaded onto Historical processes, configure [load rules](../operations/rule-configuration.md#load-rules) 
 
-The easiest way to do this is to explicitly configure the segments that don't get loaded onto Historical processes. Set `tieredReplicants` to an empty array and `useDefaultTierForNull` to `false`. For example, if you configure the following rule for a datasource:
+The easiest way to keep segments only in deep storage is to explicitly configure the segments that don't get loaded onto Historical processes. Set `tieredReplicants` to an empty array and `useDefaultTierForNull` to `false`. For example, if you configure the following rule for a datasource:
 
 ```json
 [
@@ -64,10 +71,7 @@ Segments with a `replication_factor` of `0` are not assigned to any Historical t
 
 You can also confirm this through the Druid console. On the **Segments** page, see the **Replication factor** column.
 
-Keep the following in mind when working with load rules to control what exists only in deep storage:
-
-- At least one of the segments in a datasource must be loaded onto a Historical process so that Druid can plan the query. The segment on the Historical process can be any segment from the datasource. It does not need to be a specific segment. One way to verify that a datasource has at least one segment on a Historical process is if it's visible in the Druid console.
-- The actual number of replicas may differ from the replication factor temporarily as Druid processes your load rules.
+Note that the actual number of replicas may differ from the replication factor temporarily as Druid processes your load rules.
 
 ## Run a query from deep storage
 
diff --git a/docs/tutorials/tutorial-query-deep-storage.md b/docs/tutorials/tutorial-query-deep-storage.md
index dfb4de22eb0..1bd2b96501f 100644
--- a/docs/tutorials/tutorial-query-deep-storage.md
+++ b/docs/tutorials/tutorial-query-deep-storage.md
@@ -25,7 +25,7 @@ sidebar_label: "Query from deep storage"
 
 Query from deep storage allows you to query segments that are stored only in deep storage, which provides lower costs than if you were to load everything onto Historical processes. The tradeoff is that queries from deep storage may take longer to complete. 
 
-This tutorial walks you through loading example data, configuring load rules so that not all the segments get loaded onto Historical processes, and querying data from deep storage.
+This tutorial walks you through loading example data, configuring load rules so that not all the segments get loaded onto Historical services, and querying data from deep storage. If you have [centralized datasource schema enabled](../configuration/index.md#centralized-datasource-schema), you can query datasources that are only in deep storage without having any segment available on Historical.
 
 To run the queries in this tutorial, replace `ROUTER:PORT` with the location of the Router process and its port number. For example, use `localhost:8888` for the quickstart deployment.
 
diff --git a/website/.spelling b/website/.spelling
index 3e9827e82be..74f660c2213 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -276,6 +276,7 @@ averager
 averagers
 backend
 backfills
+backfilled
 backpressure
 base64
 big-endian

From 72f8e79a42ad9a251bfc8b66c712b1c589d250a4 Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Mon, 26 Aug 2024 11:34:40 +0530
Subject: [PATCH 53/99] Use multiple workers in MSQ WF drill test suite
 (#16949)

---
 .../msq/exec/MSQDrillWindowQueryTest.java     | 103 ++++++++++++++++++
 .../sql/calcite/DrillWindowQueryTest.java     |  20 +++-
 .../multiple_over_different_sort_column.e     |  19 +---
 .../multiple_over_different_sort_column.q     |  10 +-
 .../multiple_over_multiple_sort_columns_2.e   |  80 +-------------
 .../multiple_over_multiple_sort_columns_2.q   |  10 +-
 .../multiple_over_same_sort_column.e          |  21 ++--
 .../multiple_over_same_sort_column.q          |  10 +-
 .../over_clause_only_sorting/single_over_1.e  |  21 ++--
 .../over_clause_only_sorting/single_over_1.q  |   8 +-
 .../single_over_multiple_sort_columns.q       |   2 +-
 .../partition_by_array/wikipedia_query_1.e    |  12 +-
 .../partition_by_array/wikipedia_query_1.q    |   2 +-
 .../shuffle_columns/wikipedia_query_2.q       |   2 +-
 .../wikipedia_query_2_shuffle_1.q             |   2 +-
 15 files changed, 174 insertions(+), 148 deletions(-)

diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDrillWindowQueryTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDrillWindowQueryTest.java
index 4e9821d1083..1cc756d7c57 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDrillWindowQueryTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDrillWindowQueryTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.msq.exec;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
 import com.google.inject.Injector;
 import com.google.inject.Module;
 import org.apache.druid.guice.DruidInjectorBuilder;
@@ -29,17 +30,32 @@ import org.apache.druid.msq.test.CalciteMSQTestsHelper;
 import org.apache.druid.msq.test.ExtractResultsFactory;
 import org.apache.druid.msq.test.MSQTestOverlordServiceClient;
 import org.apache.druid.msq.test.VerifyMSQSupportedNativeQueriesPredicate;
+import org.apache.druid.msq.util.MultiStageQueryContext;
+import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.groupby.TestGroupByBuffers;
 import org.apache.druid.server.QueryLifecycleFactory;
 import org.apache.druid.sql.calcite.DrillWindowQueryTest;
 import org.apache.druid.sql.calcite.QueryTestBuilder;
 import org.apache.druid.sql.calcite.SqlTestFrameworkConfig;
 import org.apache.druid.sql.calcite.TempDirProducer;
+import org.apache.druid.sql.calcite.planner.PlannerCaptureHook;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
 import org.apache.druid.sql.calcite.run.SqlEngine;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
 
 @SqlTestFrameworkConfig.ComponentSupplier(DrillWindowQueryMSQComponentSupplier.class)
 public class MSQDrillWindowQueryTest extends DrillWindowQueryTest
 {
+  private final Map<String, Object> queryContext = new HashMap<>(ImmutableMap.of(
+      PlannerContext.CTX_ENABLE_WINDOW_FNS, true,
+      PlannerCaptureHook.NEED_CAPTURE_HOOK, true,
+      QueryContexts.ENABLE_DEBUG, true,
+      MultiStageQueryContext.CTX_MAX_NUM_TASKS, 5
+  ));
+
   public static class DrillWindowQueryMSQComponentSupplier extends DrillComponentSupplier
   {
     public DrillWindowQueryMSQComponentSupplier(TempDirProducer tempFolderProducer)
@@ -74,4 +90,91 @@ public class MSQDrillWindowQueryTest extends DrillWindowQueryTest
         .skipVectorize(true)
         .verifyNativeQueries(new VerifyMSQSupportedNativeQueriesPredicate());
   }
+
+  @Override
+  protected Map<String, Object> getQueryContext()
+  {
+    return queryContext;
+  }
+
+  @Override
+  @DrillTest("druid_queries/empty_over_clause/multiple_empty_over_1")
+  @Test
+  public void test_empty_over_multiple_empty_over_1()
+  {
+    useSingleWorker();
+    windowQueryTest();
+  }
+
+  @Override
+  @DrillTest("druid_queries/empty_over_clause/single_empty_over_1")
+  @Test
+  public void test_empty_over_single_empty_over_1()
+  {
+    useSingleWorker();
+    windowQueryTest();
+  }
+
+  @Override
+  @DrillTest("druid_queries/empty_over_clause/single_empty_over_2")
+  @Test
+  public void test_empty_over_single_empty_over_2()
+  {
+    useSingleWorker();
+    windowQueryTest();
+  }
+
+  @Override
+  @DrillTest("druid_queries/empty_and_non_empty_over/wikipedia_query_1")
+  @Test
+  public void test_empty_and_non_empty_over_wikipedia_query_1()
+  {
+    useSingleWorker();
+    windowQueryTest();
+  }
+
+  @Override
+  @DrillTest("druid_queries/empty_and_non_empty_over/wikipedia_query_2")
+  @Test
+  public void test_empty_and_non_empty_over_wikipedia_query_2()
+  {
+    useSingleWorker();
+    windowQueryTest();
+  }
+
+  @Override
+  @DrillTest("druid_queries/empty_and_non_empty_over/wikipedia_query_3")
+  @Test
+  public void test_empty_and_non_empty_over_wikipedia_query_3()
+  {
+    useSingleWorker();
+    windowQueryTest();
+  }
+
+  @Override
+  @DrillTest("druid_queries/over_clause_only_partitioning/multiple_over_multiple_partition_columns_2")
+  @Test
+  public void test_over_clause_with_only_partitioning_multiple_over_multiple_partition_columns_2()
+  {
+    useSingleWorker();
+    windowQueryTest();
+  }
+
+  @Override
+  @DrillTest("druid_queries/over_clause_only_partitioning/multiple_over_different_partition_column")
+  @Test
+  public void test_over_clause_with_only_partitioning_multiple_over_different_partition_column()
+  {
+    useSingleWorker();
+    windowQueryTest();
+  }
+
+  /*
+  Queries having window functions can give multiple correct results because of using MixShuffleSpec in the previous stage.
+  So we want to use a single worker to get the same result everytime for such test cases.
+   */
+  private void useSingleWorker()
+  {
+    queryContext.put(MultiStageQueryContext.CTX_MAX_NUM_TASKS, 2);
+  }
 }
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
index c1504e8b58e..d9469aeec07 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
@@ -69,8 +69,10 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -385,14 +387,12 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
       DrillTestCase testCase = drillTestCaseRule.testCase;
       thread.setName("drillWindowQuery-" + testCase.filename);
 
+      final Map<String, Object> queryContext = new HashMap<>(testBuilder().getQueryContext());
+      queryContext.putAll(getQueryContext());
+
       testBuilder()
           .skipVectorize(true)
-          .queryContext(ImmutableMap.of(
-                            PlannerContext.CTX_ENABLE_WINDOW_FNS, true,
-                            PlannerCaptureHook.NEED_CAPTURE_HOOK, true,
-                            QueryContexts.ENABLE_DEBUG, true
-                        )
-          )
+          .queryContext(queryContext)
           .sql(testCase.getQueryString())
           .expectedResults(new TextualResultsVerifier(testCase.getExpectedResults(), null))
           .run();
@@ -404,6 +404,14 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
     }
   }
 
+  protected Map<String, Object> getQueryContext()
+  {
+    return ImmutableMap.of(
+        PlannerContext.CTX_ENABLE_WINDOW_FNS, true,
+        PlannerCaptureHook.NEED_CAPTURE_HOOK, true,
+        QueryContexts.ENABLE_DEBUG, true
+    );
+  }
 
   // testcases_start
   @DrillTest("aggregates/aggOWnFn_11")
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_different_sort_column.e b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_different_sort_column.e
index 151090f3be2..3e592e4b901 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_different_sort_column.e
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_different_sort_column.e
@@ -1,13 +1,6 @@
-Austria	null	#de.wikipedia	1	null
-Austria	Horsching	#de.wikipedia	2	null
-Austria	Vienna	#de.wikipedia	3	Horsching
-Austria	Vienna	#es.wikipedia	5	Vienna
-Austria	Vienna	#tr.wikipedia	13	Vienna
-Republic of Korea	null	#en.wikipedia	4	Vienna
-Republic of Korea	null	#ja.wikipedia	6	null
-Republic of Korea	null	#ko.wikipedia	7	null
-Republic of Korea	Jeonju	#ko.wikipedia	8	null
-Republic of Korea	Seongnam-si	#ko.wikipedia	9	Jeonju
-Republic of Korea	Seoul	#ko.wikipedia	10	Seongnam-si
-Republic of Korea	Suwon-si	#ko.wikipedia	11	Seoul
-Republic of Korea	Yongsan-dong	#ko.wikipedia	12	Suwon-si
+Guatemala	null	Sport Lisboa e Benfica	-167	5	null
+El Salvador	San Salvador	Wendigo	-1	6	null
+Guatemala	El Salvador	Fuerzas Armadas de Honduras	1	3	San Salvador
+El Salvador	San Salvador	Pedro de Valdivia	9	4	El Salvador
+El Salvador	San Salvador	Amor de barrio	106	1	San Salvador
+Guatemala	Guatemala City	Anexo:Segunda temporada de Gravity Falls	173	2	San Salvador
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_different_sort_column.q b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_different_sort_column.q
index 886bc0ac11f..73f07f1a3c5 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_different_sort_column.q
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_different_sort_column.q
@@ -1,7 +1,7 @@
 select
-countryName, cityName, channel,
-row_number() over (order by channel) as c1,
-lag(cityName) over (order by countryName) as c2
+countryName, cityName, page, delta,
+row_number() over (order by page) as c1,
+lag(cityName) over (order by delta) as c2
 from wikipedia
-where countryName in ('Austria', 'Republic of Korea')
-group by countryName, cityName, channel
+where countryName in ('Guatemala', 'El Salvador')
+group by countryName, cityName, page, delta
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_multiple_sort_columns_2.e b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_multiple_sort_columns_2.e
index 6710f17c2a9..3ab4c941317 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_multiple_sort_columns_2.e
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_multiple_sort_columns_2.e
@@ -1,74 +1,6 @@
-Austria	null	#de.wikipedia	0	1	null
-Austria	Horsching	#de.wikipedia	0	7	null
-Austria	Vienna	#de.wikipedia	0	8	Horsching
-Austria	null	#de.wikipedia	14	2	Vienna
-Austria	Vienna	#de.wikipedia	72	9	null
-Austria	null	#de.wikipedia	94	3	Vienna
-Austria	null	#de.wikipedia	272	4	null
-Austria	null	#de.wikipedia	4685	5	null
-Austria	null	#de.wikipedia	6979	6	null
-Republic of Korea	null	#en.wikipedia	0	12	null
-Republic of Korea	null	#en.wikipedia	5	13	null
-Republic of Korea	null	#en.wikipedia	75	14	null
-Austria	Vienna	#es.wikipedia	0	10	null
-Republic of Korea	null	#ja.wikipedia	0	15	Vienna
-Republic of Korea	null	#ja.wikipedia	43	16	null
-Republic of Korea	null	#ja.wikipedia	46	17	null
-Republic of Korea	null	#ja.wikipedia	356	18	null
-Republic of Korea	null	#ko.wikipedia	0	19	null
-Republic of Korea	Seoul	#ko.wikipedia	0	59	null
-Republic of Korea	Yongsan-dong	#ko.wikipedia	0	74	Seoul
-Republic of Korea	null	#ko.wikipedia	1	20	Yongsan-dong
-Republic of Korea	Jeonju	#ko.wikipedia	1	57	null
-Republic of Korea	null	#ko.wikipedia	2	21	Jeonju
-Republic of Korea	Seoul	#ko.wikipedia	2	60	null
-Republic of Korea	null	#ko.wikipedia	3	22	Seoul
-Republic of Korea	null	#ko.wikipedia	4	23	null
-Republic of Korea	Seoul	#ko.wikipedia	4	61	null
-Republic of Korea	Seoul	#ko.wikipedia	5	62	Seoul
-Republic of Korea	Seoul	#ko.wikipedia	6	63	Seoul
-Republic of Korea	null	#ko.wikipedia	8	24	Seoul
-Republic of Korea	null	#ko.wikipedia	11	25	null
-Republic of Korea	Seoul	#ko.wikipedia	12	64	null
-Republic of Korea	null	#ko.wikipedia	13	26	Seoul
-Republic of Korea	null	#ko.wikipedia	14	27	null
-Republic of Korea	Seoul	#ko.wikipedia	15	65	null
-Republic of Korea	null	#ko.wikipedia	19	28	Seoul
-Republic of Korea	Seoul	#ko.wikipedia	19	66	null
-Republic of Korea	null	#ko.wikipedia	22	29	Seoul
-Republic of Korea	Seoul	#ko.wikipedia	22	67	null
-Republic of Korea	null	#ko.wikipedia	23	30	Seoul
-Republic of Korea	null	#ko.wikipedia	24	31	null
-Republic of Korea	Seoul	#ko.wikipedia	24	68	null
-Republic of Korea	null	#ko.wikipedia	26	32	Seoul
-Republic of Korea	null	#ko.wikipedia	29	33	null
-Republic of Korea	null	#ko.wikipedia	30	34	null
-Republic of Korea	null	#ko.wikipedia	34	35	null
-Republic of Korea	Suwon-si	#ko.wikipedia	40	72	null
-Republic of Korea	null	#ko.wikipedia	41	36	Suwon-si
-Republic of Korea	null	#ko.wikipedia	42	37	null
-Republic of Korea	null	#ko.wikipedia	51	38	null
-Republic of Korea	null	#ko.wikipedia	52	39	null
-Republic of Korea	null	#ko.wikipedia	56	40	null
-Republic of Korea	null	#ko.wikipedia	59	41	null
-Republic of Korea	null	#ko.wikipedia	62	42	null
-Republic of Korea	null	#ko.wikipedia	65	43	null
-Republic of Korea	null	#ko.wikipedia	73	44	null
-Republic of Korea	null	#ko.wikipedia	159	45	null
-Republic of Korea	Seoul	#ko.wikipedia	243	69	null
-Republic of Korea	Seongnam-si	#ko.wikipedia	254	58	Seoul
-Republic of Korea	null	#ko.wikipedia	357	46	Seongnam-si
-Republic of Korea	null	#ko.wikipedia	390	47	null
-Republic of Korea	null	#ko.wikipedia	400	48	null
-Republic of Korea	null	#ko.wikipedia	495	49	null
-Republic of Korea	null	#ko.wikipedia	524	50	null
-Republic of Korea	null	#ko.wikipedia	566	51	null
-Republic of Korea	Seoul	#ko.wikipedia	663	70	null
-Republic of Korea	null	#ko.wikipedia	757	52	Seoul
-Republic of Korea	null	#ko.wikipedia	827	53	null
-Republic of Korea	null	#ko.wikipedia	832	54	null
-Republic of Korea	null	#ko.wikipedia	1006	55	null
-Republic of Korea	Seoul	#ko.wikipedia	1036	71	null
-Republic of Korea	null	#ko.wikipedia	2539	56	Seoul
-Republic of Korea	Suwon-si	#ko.wikipedia	3234	73	null
-Austria	Vienna	#tr.wikipedia	93	11	Suwon-si
+Guatemala	null	#es.wikipedia	0	Sport Lisboa e Benfica	-167	4	null
+El Salvador	San Salvador	#es.wikipedia	0	Wendigo	-1	3	null
+Guatemala	El Salvador	#es.wikipedia	1	Fuerzas Armadas de Honduras	1	5	San Salvador
+El Salvador	San Salvador	#es.wikipedia	9	Pedro de Valdivia	9	2	El Salvador
+El Salvador	San Salvador	#es.wikipedia	106	Amor de barrio	106	1	San Salvador
+Guatemala	Guatemala City	#es.wikipedia	173	Anexo:Segunda temporada de Gravity Falls	173	6	San Salvador
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_multiple_sort_columns_2.q b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_multiple_sort_columns_2.q
index e2608974f8d..c804b60b36b 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_multiple_sort_columns_2.q
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_multiple_sort_columns_2.q
@@ -1,7 +1,7 @@
 select
-countryName, cityName, channel, added,
-row_number() over (order by countryName, cityName) as c1,
-lag(cityName) over (order by channel, added) as c2
+countryName, cityName, channel, added, page, delta,
+row_number() over (order by countryName, cityName, page) as c1,
+lag(cityName) over (order by channel, added, delta) as c2
 from wikipedia
-where countryName in ('Austria', 'Republic of Korea')
-group by countryName, cityName, channel, added
+where countryName in ('Guatemala', 'El Salvador')
+group by countryName, cityName, channel, added, page, delta
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_same_sort_column.e b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_same_sort_column.e
index bfe949f40f5..be09afabea5 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_same_sort_column.e
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_same_sort_column.e
@@ -1,13 +1,8 @@
-Austria	null	#de.wikipedia	1	null
-Austria	Horsching	#de.wikipedia	2	null
-Austria	Vienna	#de.wikipedia	3	Horsching
-Republic of Korea	null	#en.wikipedia	4	Vienna
-Austria	Vienna	#es.wikipedia	5	null
-Republic of Korea	null	#ja.wikipedia	6	Vienna
-Republic of Korea	null	#ko.wikipedia	7	null
-Republic of Korea	Jeonju	#ko.wikipedia	8	null
-Republic of Korea	Seongnam-si	#ko.wikipedia	9	Jeonju
-Republic of Korea	Seoul	#ko.wikipedia	10	Seongnam-si
-Republic of Korea	Suwon-si	#ko.wikipedia	11	Seoul
-Republic of Korea	Yongsan-dong	#ko.wikipedia	12	Suwon-si
-Austria	Vienna	#tr.wikipedia	13	Yongsan-dong
+El Salvador	San Salvador	0	1	null
+Egypt	null	1	2	San Salvador
+El Salvador	San Salvador	9	3	null
+Egypt	Cairo	14	4	San Salvador
+Egypt	null	16	5	Cairo
+Egypt	null	27	6	null
+El Salvador	San Salvador	106	7	null
+Egypt	null	112	8	San Salvador
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_same_sort_column.q b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_same_sort_column.q
index 723e0ea2963..e7f09dcb7a7 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_same_sort_column.q
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/multiple_over_same_sort_column.q
@@ -1,7 +1,7 @@
 select
-countryName, cityName, channel,
-row_number() over (order by channel) as c1,
-lag(cityName) over (order by channel) as c2
+countryName, cityName, added,
+row_number() over (order by added) as c1,
+lag(cityName) over (order by added) as c2
 from wikipedia
-where countryName in ('Austria', 'Republic of Korea')
-group by countryName, cityName, channel
+where countryName in ('Egypt', 'El Salvador')
+group by countryName, cityName, added
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_1.e b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_1.e
index 3b1046264ef..0af49de91e7 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_1.e
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_1.e
@@ -1,13 +1,8 @@
-Austria	null	1
-Republic of Korea	null	2
-Republic of Korea	null	3
-Republic of Korea	null	4
-Austria	Horsching	5
-Republic of Korea	Jeonju	6
-Republic of Korea	Seongnam-si	7
-Republic of Korea	Seoul	8
-Republic of Korea	Suwon-si	9
-Austria	Vienna	10
-Austria	Vienna	11
-Austria	Vienna	12
-Republic of Korea	Yongsan-dong	13
+El Salvador	0	1
+Egypt	1	2
+El Salvador	9	3
+Egypt	14	4
+Egypt	16	5
+Egypt	27	6
+El Salvador	106	7
+Egypt	112	8
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_1.q b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_1.q
index 10b81e433b1..477c2be04cf 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_1.q
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_1.q
@@ -1,6 +1,6 @@
 select
-countryName, cityName,
-row_number() over (order by cityName) as c1
+countryName, added,
+row_number() over (order by added) as c1
 from wikipedia
-where countryName in ('Austria', 'Republic of Korea')
-group by countryName, cityName, channel
+where countryName in ('Egypt', 'El Salvador')
+group by countryName, cityName, added
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_multiple_sort_columns.q b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_multiple_sort_columns.q
index edce01210da..a4a40a94a77 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_multiple_sort_columns.q
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/over_clause_only_sorting/single_over_multiple_sort_columns.q
@@ -1,6 +1,6 @@
 select
 countryName, cityName, channel,
-lag(cityName) over (order by channel, countryName) as c1
+lag(cityName) over (order by channel, countryName, cityName) as c1
 from wikipedia
 where countryName in ('Austria', 'Republic of Korea')
 group by countryName, cityName, channel
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/partition_by_array/wikipedia_query_1.e b/sql/src/test/resources/drill/window/queries/druid_queries/partition_by_array/wikipedia_query_1.e
index 26c251a35fb..8dcf46409bc 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/partition_by_array/wikipedia_query_1.e
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/partition_by_array/wikipedia_query_1.e
@@ -1,3 +1,7 @@
+Republic of Korea	Yongsan-dong	#ko.wikipedia	1
+Republic of Korea	Suwon-si	#ko.wikipedia	1
+Austria	Horsching	#de.wikipedia	1
+Republic of Korea	Seongnam-si	#ko.wikipedia	1
 Austria	null	#de.wikipedia	1
 Republic of Korea	null	#en.wikipedia	2
 Republic of Korea	null	#ja.wikipedia	3
@@ -5,9 +9,5 @@ Republic of Korea	null	#ko.wikipedia	4
 Republic of Korea	Seoul	#ko.wikipedia	1
 Austria	Vienna	#de.wikipedia	1
 Austria	Vienna	#es.wikipedia	2
-Austria	Vienna	#tr.wikipedia	3
-Republic of Korea	Jeonju	#ko.wikipedia	4
-Republic of Korea	Suwon-si	#ko.wikipedia	1
-Austria	Horsching	#de.wikipedia	1
-Republic of Korea	Seongnam-si	#ko.wikipedia	1
-Republic of Korea	Yongsan-dong	#ko.wikipedia	1
+Republic of Korea	Jeonju	#ko.wikipedia	3
+Austria	Vienna	#tr.wikipedia	4
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/partition_by_array/wikipedia_query_1.q b/sql/src/test/resources/drill/window/queries/druid_queries/partition_by_array/wikipedia_query_1.q
index b10b52af389..41f1c5c6303 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/partition_by_array/wikipedia_query_1.q
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/partition_by_array/wikipedia_query_1.q
@@ -1,6 +1,6 @@
 select
 countryName, cityName, channel,
-row_number() over (partition by array[1,2,length(cityName)] order by countryName) as c
+row_number() over (partition by array[1,2,length(cityName)] order by channel) as c
 from wikipedia
 where countryName in ('Austria', 'Republic of Korea')
 group by countryName, cityName, channel
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/shuffle_columns/wikipedia_query_2.q b/sql/src/test/resources/drill/window/queries/druid_queries/shuffle_columns/wikipedia_query_2.q
index d3ea2dfc729..15d56ffdac4 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/shuffle_columns/wikipedia_query_2.q
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/shuffle_columns/wikipedia_query_2.q
@@ -1,7 +1,7 @@
 SELECT
 countryName,
 cityName,
-ROW_NUMBER() OVER(PARTITION BY countryName),
+ROW_NUMBER() OVER(PARTITION BY countryName order by countryName, cityName, channel),
 channel,
 COUNT(channel) over (PARTITION BY cityName order by countryName, cityName, channel)
 FROM wikipedia
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/shuffle_columns/wikipedia_query_2_shuffle_1.q b/sql/src/test/resources/drill/window/queries/druid_queries/shuffle_columns/wikipedia_query_2_shuffle_1.q
index 779aaf3a86f..8b354f37540 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/shuffle_columns/wikipedia_query_2_shuffle_1.q
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/shuffle_columns/wikipedia_query_2_shuffle_1.q
@@ -1,5 +1,5 @@
 SELECT
-ROW_NUMBER() OVER(PARTITION BY countryName),
+ROW_NUMBER() OVER(PARTITION BY countryName order by countryName, cityName, channel),
 countryName,
 cityName,
 COUNT(channel) over (PARTITION BY cityName order by countryName, cityName, channel),

From 7ee7e194c4ee3979d353df5af2854808644cbd4e Mon Sep 17 00:00:00 2001
From: George Shiqi Wu <george.wu@imply.io>
Date: Mon, 26 Aug 2024 07:40:02 -0700
Subject: [PATCH 54/99] Add supervisor log when task count is greater than
 partitions (#16948)

* Add log message when task count is higher than partitions

* newline

* fix ordering

* Add supervisor id

* Update indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java

Co-authored-by: Kashif Faraz <kashif.faraz@gmail.com>

* Update indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java

Co-authored-by: Kashif Faraz <kashif.faraz@gmail.com>

* Update indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java

Co-authored-by: Kashif Faraz <kashif.faraz@gmail.com>

---------

Co-authored-by: Kashif Faraz <kashif.faraz@gmail.com>
---
 .../seekablestream/supervisor/SeekableStreamSupervisor.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
index 81d8871b8e8..d7294abc9d5 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
@@ -2733,6 +2733,10 @@ public abstract class SeekableStreamSupervisor<PartitionIdType, SequenceOffsetTy
 
     log.debug("Found [%d] partitions for stream [%s]", partitionIdsFromSupplier.size(), ioConfig.getStream());
 
+    final int configuredTaskCount = spec.getIoConfig().getTaskCount();
+    if (configuredTaskCount > partitionIdsFromSupplier.size()) {
+      log.warn("Configured task count[%s] for supervisor[%s] is greater than the number of partitions[%d].", configuredTaskCount, supervisorId, partitionIdsFromSupplier.size());
+    }
     Map<PartitionIdType, SequenceOffsetType> storedMetadata = getOffsetsFromMetadataStorage();
     Set<PartitionIdType> storedPartitions = storedMetadata.keySet();
     Set<PartitionIdType> closedPartitions = storedMetadata

From 5d2ed33b8975405916aba91fb91c8b408562de94 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Mon, 26 Aug 2024 21:45:51 -0700
Subject: [PATCH 55/99] Place __time in signatures according to sort order.
 (#16958)

* Place __time in signatures according to sort order.

Updates a variety of places to put __time in row signatures according
to its position in the sort order, rather than always first, including:

- InputSourceSampler.
- ScanQueryEngine (in the default signature when "columns" is empty).
- Various StorageAdapters, which also have the effect of reordering
  the column order in segmentMetadata queries, and therefore in SQL
  schemas as well.

Follow-up to #16849.

* Fix compilation.

* Additional fixes.

* Fix.

* Fix style.

* Omit nonexistent columns from the row signature.

* Fix tests.
---
 .../apache/druid/msq/exec/MSQReplaceTest.java | 28 +++++++-------
 .../apache/druid/msq/test/MSQTestBase.java    | 22 ++++++-----
 .../overlord/sampler/InputSourceSampler.java  | 16 ++++----
 .../druid/query/scan/ScanQueryEngine.java     | 13 ++-----
 .../druid/segment/FilteredStorageAdapter.java |  7 ++++
 .../segment/QueryableIndexStorageAdapter.java | 37 +++++++++++++++++++
 .../apache/druid/segment/StorageAdapter.java  |  7 +---
 .../druid/segment/UnnestStorageAdapter.java   | 18 +++++++++
 .../druid/segment/column/ColumnType.java      |  6 +++
 .../IncrementalIndexStorageAdapter.java       | 14 +++++++
 .../join/HashJoinSegmentStorageAdapter.java   | 20 ++++++++++
 .../table/BroadcastSegmentIndexedTable.java   |  8 +---
 .../apache/druid/frame/key/KeyTestUtils.java  | 10 ++---
 .../druid/query/lookup/LookupSegmentTest.java | 14 +++++++
 .../druid/segment/IndexMergerTestBase.java    | 20 +++++-----
 .../HashJoinSegmentStorageAdapterTest.java    | 25 +++++++++++++
 .../BroadcastSegmentIndexedTableTest.java     |  6 +--
 .../druid/segment/realtime/sink/Sink.java     | 23 ++++++------
 18 files changed, 211 insertions(+), 83 deletions(-)

diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
index 96da93c34d0..e470a951877 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
@@ -279,8 +279,8 @@ public class MSQReplaceTest extends MSQTestBase
   public void testReplaceOnFooWithAllClusteredByDimExplicitSort(String contextName, Map<String, Object> context)
   {
     RowSignature rowSignature = RowSignature.builder()
-                                            .add("__time", ColumnType.LONG)
                                             .add("dim1", ColumnType.STRING)
+                                            .add("__time", ColumnType.LONG)
                                             .add("m1", ColumnType.FLOAT)
                                             .build();
 
@@ -323,12 +323,12 @@ public class MSQReplaceTest extends MSQTestBase
                      .setExpectedShardSpec(DimensionRangeShardSpec.class)
                      .setExpectedResultRows(
                          ImmutableList.of(
-                             new Object[]{946684800000L, NullHandling.sqlCompatible() ? "" : null, 1.0f},
-                             new Object[]{978307200000L, "1", 4.0f},
-                             new Object[]{946771200000L, "10.1", 2.0f},
-                             new Object[]{946857600000L, "2", 3.0f},
-                             new Object[]{978480000000L, "abc", 6.0f},
-                             new Object[]{978393600000L, "def", 5.0f}
+                             new Object[]{NullHandling.sqlCompatible() ? "" : null, 946684800000L, 1.0f},
+                             new Object[]{"1", 978307200000L, 4.0f},
+                             new Object[]{"10.1", 946771200000L, 2.0f},
+                             new Object[]{"2", 946857600000L, 3.0f},
+                             new Object[]{"abc", 978480000000L, 6.0f},
+                             new Object[]{"def", 978393600000L, 5.0f}
                          )
                      )
                      .setExpectedSegmentGenerationProgressCountersForStageWorker(
@@ -365,8 +365,8 @@ public class MSQReplaceTest extends MSQTestBase
     // forceSegmentSortByTime = false. (Same expectations as the prior test,
     // testReplaceOnFooWithAllClusteredByDimExplicitSort.)
     RowSignature rowSignature = RowSignature.builder()
-                                            .add("__time", ColumnType.LONG)
                                             .add("dim1", ColumnType.STRING)
+                                            .add("__time", ColumnType.LONG)
                                             .add("m1", ColumnType.FLOAT)
                                             .build();
 
@@ -409,12 +409,12 @@ public class MSQReplaceTest extends MSQTestBase
                      .setExpectedShardSpec(DimensionRangeShardSpec.class)
                      .setExpectedResultRows(
                          ImmutableList.of(
-                             new Object[]{946684800000L, NullHandling.sqlCompatible() ? "" : null, 1.0f},
-                             new Object[]{978307200000L, "1", 4.0f},
-                             new Object[]{946771200000L, "10.1", 2.0f},
-                             new Object[]{946857600000L, "2", 3.0f},
-                             new Object[]{978480000000L, "abc", 6.0f},
-                             new Object[]{978393600000L, "def", 5.0f}
+                             new Object[]{NullHandling.sqlCompatible() ? "" : null, 946684800000L, 1.0f},
+                             new Object[]{"1", 978307200000L, 4.0f},
+                             new Object[]{"10.1", 946771200000L, 2.0f},
+                             new Object[]{"2", 946857600000L, 3.0f},
+                             new Object[]{"abc", 978480000000L, 6.0f},
+                             new Object[]{"def", 978393600000L, 5.0f}
                          )
                      )
                      .setExpectedSegmentGenerationProgressCountersForStageWorker(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
index 5076e9f3d44..258b5c97751 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
@@ -141,6 +141,7 @@ import org.apache.druid.segment.Segment;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.TestIndex;
 import org.apache.druid.segment.column.ColumnConfig;
+import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.incremental.IncrementalIndexSchema;
 import org.apache.druid.segment.loading.DataSegmentPusher;
@@ -1288,17 +1289,20 @@ public class MSQTestBase extends BaseCalciteQueryTest
           Assert.assertEquals(expectedDestinationIntervals, destination.getReplaceTimeChunks());
         }
         if (expectedSegments != null) {
+          final int timeIndex =
+              MSQResultsReport.ColumnAndType.toRowSignature(expectedRowSignature)
+                                            .indexOf(ColumnHolder.TIME_COLUMN_NAME);
           Assert.assertEquals(expectedSegments, segmentIdVsOutputRowsMap.keySet());
           for (Object[] row : transformedOutputRows) {
-            List<SegmentId> diskSegmentList = segmentIdVsOutputRowsMap.keySet()
-                                                                      .stream()
-                                                                      .filter(segmentId -> segmentId.getInterval()
-                                                                                                    .contains((Long) row[0]))
-                                                                      .filter(segmentId -> {
-                                                                        List<List<Object>> lists = segmentIdVsOutputRowsMap.get(segmentId);
-                                                                        return lists.contains(Arrays.asList(row));
-                                                                      })
-                                                                      .collect(Collectors.toList());
+            List<SegmentId> diskSegmentList = segmentIdVsOutputRowsMap
+                .keySet()
+                .stream()
+                .filter(segmentId -> segmentId.getInterval().contains((Long) row[timeIndex]))
+                .filter(segmentId -> {
+                  List<List<Object>> lists = segmentIdVsOutputRowsMap.get(segmentId);
+                  return lists.contains(Arrays.asList(row));
+                })
+                .collect(Collectors.toList());
             if (diskSegmentList.size() != 1) {
               throw new IllegalStateException("Single key in multiple partitions");
             }
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java
index 4e5e8634f18..f98287124ed 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java
@@ -241,14 +241,12 @@ public class InputSourceSampler
         List<DimensionSchema> physicalDimensionSchemas = new ArrayList<>();
 
         RowSignature.Builder signatureBuilder = RowSignature.builder();
-        signatureBuilder.add(
-            ColumnHolder.TIME_COLUMN_NAME,
-            index.getColumnCapabilities(ColumnHolder.TIME_COLUMN_NAME).toColumnType()
-        );
-        for (IncrementalIndex.DimensionDesc dimensionDesc : index.getDimensions()) {
-          if (!SamplerInputRow.SAMPLER_ORDERING_COLUMN.equals(dimensionDesc.getName())) {
-            final ColumnType columnType = dimensionDesc.getCapabilities().toColumnType();
-            signatureBuilder.add(dimensionDesc.getName(), columnType);
+        for (final String dimensionName : index.getDimensionNames(true)) {
+          if (ColumnHolder.TIME_COLUMN_NAME.equals(dimensionName)) {
+            signatureBuilder.addTimeColumn();
+          } else if (!SamplerInputRow.SAMPLER_ORDERING_COLUMN.equals(dimensionName)) {
+            final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensionName);
+            signatureBuilder.add(dimensionDesc.getName(), ColumnType.fromCapabilities(dimensionDesc.getCapabilities()));
             // use explicitly specified dimension schema if it exists
             if (dataSchema != null &&
                 dataSchema.getDimensionsSpec() != null &&
@@ -271,7 +269,7 @@ public class InputSourceSampler
           if (!SamplerInputRow.SAMPLER_ORDERING_COLUMN.equals(aggregatorFactory.getName())) {
             signatureBuilder.add(
                 aggregatorFactory.getName(),
-                index.getColumnCapabilities(aggregatorFactory.getName()).toColumnType()
+                ColumnType.fromCapabilities(index.getColumnCapabilities(aggregatorFactory.getName()))
             );
           }
         }
diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
index ca2c2725c6f..9dd72d15fa4 100644
--- a/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQueryEngine.java
@@ -45,6 +45,7 @@ import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumn;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.filter.Filters;
 import org.apache.druid.timeline.SegmentId;
@@ -98,13 +99,11 @@ public class ScanQueryEngine
     } else {
       final Set<String> availableColumns = Sets.newLinkedHashSet(
           Iterables.concat(
-              Collections.singleton(ColumnHolder.TIME_COLUMN_NAME),
+              adapter.getRowSignature().getColumnNames(),
               Iterables.transform(
                   Arrays.asList(query.getVirtualColumns().getVirtualColumns()),
                   VirtualColumn::getOutputName
-              ),
-              adapter.getAvailableDimensions(),
-              adapter.getAvailableMetrics()
+              )
           )
       );
 
@@ -152,11 +151,7 @@ public class ScanQueryEngine
             for (String column : allColumns) {
               final BaseObjectColumnValueSelector selector = factory.makeColumnValueSelector(column);
               ColumnCapabilities columnCapabilities = factory.getColumnCapabilities(column);
-              rowSignatureBuilder.add(
-                  column,
-                  columnCapabilities == null ? null : columnCapabilities.toColumnType()
-              );
-
+              rowSignatureBuilder.add(column, ColumnType.fromCapabilities(columnCapabilities));
               columnSelectors.add(selector);
             }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
index 3fcfe2dd78a..46b9f5fe061 100644
--- a/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/FilteredStorageAdapter.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.filter.AndFilter;
 import org.joda.time.Interval;
@@ -69,6 +70,12 @@ public class FilteredStorageAdapter implements StorageAdapter
     return baseStorageAdapter.getInterval();
   }
 
+  @Override
+  public RowSignature getRowSignature()
+  {
+    return baseStorageAdapter.getRowSignature();
+  }
+
   @Override
   public Indexed<String> getAvailableDimensions()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
index e2c88071607..a0e23e71f74 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
@@ -19,11 +19,14 @@
 
 package org.apache.druid.segment;
 
+import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.column.BaseColumn;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnIndexSupplier;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.DictionaryEncodedColumn;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex;
 import org.joda.time.DateTime;
@@ -55,6 +58,40 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
     return index.getDataInterval();
   }
 
+  @Override
+  public RowSignature getRowSignature()
+  {
+    final LinkedHashSet<String> columns = new LinkedHashSet<>();
+
+    for (final OrderBy orderBy : index.getOrdering()) {
+      columns.add(orderBy.getColumnName());
+    }
+
+    // Add __time after the defined ordering, if __time wasn't part of it.
+    columns.add(ColumnHolder.TIME_COLUMN_NAME);
+
+    for (final String dimension : getAvailableDimensions()) {
+      columns.add(dimension);
+    }
+
+    for (final String metric : getAvailableMetrics()) {
+      columns.add(metric);
+    }
+
+    final RowSignature.Builder builder = RowSignature.builder();
+    for (final String column : columns) {
+      final ColumnType columnType = ColumnType.fromCapabilities(index.getColumnCapabilities(column));
+
+      // index.getOrdering() may include columns that don't exist, such as if they were omitted due to
+      // being 100% nulls. Don't add those to the row signature.
+      if (columnType != null) {
+        builder.add(column, columnType);
+      }
+    }
+
+    return builder.build();
+  }
+
   @Override
   public Indexed<String> getAvailableDimensions()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
index 3ad6c930002..c0949692e4f 100644
--- a/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java
@@ -26,6 +26,7 @@ import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.query.OrderBy;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.vector.VectorCursor;
@@ -34,7 +35,6 @@ import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
 import java.util.List;
-import java.util.Optional;
 
 /**
  *
@@ -140,10 +140,7 @@ public interface StorageAdapter extends CursorFactory, ColumnInspector, CursorHo
     builder.addTimeColumn();
 
     for (final String column : Iterables.concat(getAvailableDimensions(), getAvailableMetrics())) {
-      builder.add(
-          column,
-          Optional.ofNullable(getColumnCapabilities(column)).map(ColumnCapabilities::toColumnType).orElse(null)
-      );
+      builder.add(column, ColumnType.fromCapabilities(getColumnCapabilities(column)));
     }
 
     return builder.build();
diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
index 435771b4780..2f9552a1d3c 100644
--- a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
@@ -35,6 +35,8 @@ import org.apache.druid.query.filter.NullFilter;
 import org.apache.druid.query.filter.RangeFilter;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.TypeSignature;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.data.Indexed;
@@ -169,6 +171,22 @@ public class UnnestStorageAdapter implements StorageAdapter
     return baseAdapter.getInterval();
   }
 
+  @Override
+  public RowSignature getRowSignature()
+  {
+    final RowSignature.Builder builder = RowSignature.builder();
+
+    final RowSignature baseSignature = baseAdapter.getRowSignature();
+    for (int i = 0; i < baseSignature.size(); i++) {
+      final String column = baseSignature.getColumnName(i);
+      if (!outputColumnName.equals(column)) {
+        builder.add(column, ColumnType.fromCapabilities(getColumnCapabilities(column)));
+      }
+    }
+
+    return builder.add(outputColumnName, ColumnType.fromCapabilities(getColumnCapabilities(outputColumnName))).build();
+  }
+
   @Override
   public Indexed<String> getAvailableDimensions()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/column/ColumnType.java b/processing/src/main/java/org/apache/druid/segment/column/ColumnType.java
index dbfed07749c..b670d8f4370 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/ColumnType.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/ColumnType.java
@@ -124,6 +124,12 @@ public class ColumnType extends BaseTypeSignature<ValueType>
     return Types.fromString(ColumnTypeFactory.getInstance(), typeName);
   }
 
+  @Nullable
+  public static ColumnType fromCapabilities(@Nullable ColumnCapabilities capabilities)
+  {
+    return capabilities != null ? capabilities.toColumnType() : null;
+  }
+
   public static ColumnType ofArray(ColumnType elementType)
   {
     return ColumnTypeFactory.getInstance().ofArray(elementType);
diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
index ab92a00897b..c9a6d209697 100644
--- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.segment.incremental;
 
+import com.google.common.collect.Iterables;
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionDictionarySelector;
@@ -30,6 +31,7 @@ import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.ListIndexed;
 import org.joda.time.Interval;
@@ -122,6 +124,18 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
     return index.getInterval();
   }
 
+  @Override
+  public RowSignature getRowSignature()
+  {
+    final RowSignature.Builder builder = RowSignature.builder();
+
+    for (final String column : Iterables.concat(index.getDimensionNames(true), index.getMetricNames())) {
+      builder.add(column, ColumnType.fromCapabilities(index.getColumnCapabilities(column)));
+    }
+
+    return builder.build();
+  }
+
   @Override
   public Indexed<String> getAvailableDimensions()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
index f3668ca45e1..765ab4fbd91 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapter.java
@@ -33,6 +33,8 @@ import org.apache.druid.segment.Metadata;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.ListIndexed;
 import org.apache.druid.segment.filter.Filters;
@@ -99,6 +101,24 @@ public class HashJoinSegmentStorageAdapter implements StorageAdapter
     return baseAdapter.getInterval();
   }
 
+  @Override
+  public RowSignature getRowSignature()
+  {
+    // Use a Set since we may encounter duplicates, if a field from a Joinable shadows one of the base fields.
+    final LinkedHashSet<String> columns = new LinkedHashSet<>(baseAdapter.getRowSignature().getColumnNames());
+
+    for (final JoinableClause clause : clauses) {
+      columns.addAll(clause.getAvailableColumnsPrefixed());
+    }
+
+    final RowSignature.Builder builder = RowSignature.builder();
+    for (final String column : columns) {
+      builder.add(column, ColumnType.fromCapabilities(getColumnCapabilities(column)));
+    }
+
+    return builder.build();
+  }
+
   @Override
   public Indexed<String> getAvailableDimensions()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java b/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
index 9591dc315f0..5fe6d8a698a 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTable.java
@@ -39,7 +39,6 @@ import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.SimpleAscendingOffset;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.BaseColumn;
-import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.data.ReadableOffset;
@@ -88,12 +87,7 @@ public class BroadcastSegmentIndexedTable implements IndexedTable
         segment.getId()
     );
 
-    RowSignature.Builder sigBuilder = RowSignature.builder();
-    sigBuilder.add(ColumnHolder.TIME_COLUMN_NAME, ColumnType.LONG);
-    for (String column : queryableIndex.getColumnNames()) {
-      sigBuilder.add(column, adapter.getColumnCapabilities(column).toColumnType());
-    }
-    this.rowSignature = sigBuilder.build();
+    this.rowSignature = adapter.getRowSignature();
 
     // initialize keycolumn index builders
     final ArrayList<RowBasedIndexBuilder> indexBuilders = new ArrayList<>(rowSignature.size());
diff --git a/processing/src/test/java/org/apache/druid/frame/key/KeyTestUtils.java b/processing/src/test/java/org/apache/druid/frame/key/KeyTestUtils.java
index e5ab3c203f4..cae9439a331 100644
--- a/processing/src/test/java/org/apache/druid/frame/key/KeyTestUtils.java
+++ b/processing/src/test/java/org/apache/druid/frame/key/KeyTestUtils.java
@@ -30,13 +30,11 @@ import org.apache.druid.frame.write.FrameWriters;
 import org.apache.druid.frame.write.RowBasedFrameWriter;
 import org.apache.druid.segment.ColumnInspector;
 import org.apache.druid.segment.RowBasedColumnSelectorFactory;
-import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 
 import java.util.Collections;
 import java.util.List;
-import java.util.Optional;
 
 public class KeyTestUtils
 {
@@ -56,10 +54,10 @@ public class KeyTestUtils
     final RowSignature.Builder builder = RowSignature.builder();
 
     for (final KeyColumn keyColumn : keyColumns) {
-      final ColumnCapabilities capabilities = inspector.getColumnCapabilities(keyColumn.columnName());
-      final ColumnType columnType =
-          Optional.ofNullable(capabilities).map(ColumnCapabilities::toColumnType).orElse(null);
-      builder.add(keyColumn.columnName(), columnType);
+      builder.add(
+          keyColumn.columnName(),
+          ColumnType.fromCapabilities(inspector.getColumnCapabilities(keyColumn.columnName()))
+      );
     }
 
     return builder.build();
diff --git a/processing/src/test/java/org/apache/druid/query/lookup/LookupSegmentTest.java b/processing/src/test/java/org/apache/druid/query/lookup/LookupSegmentTest.java
index 5826301bcd7..142b23b8391 100644
--- a/processing/src/test/java/org/apache/druid/query/lookup/LookupSegmentTest.java
+++ b/processing/src/test/java/org/apache/druid/query/lookup/LookupSegmentTest.java
@@ -32,6 +32,8 @@ import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.DimensionDictionarySelector;
 import org.apache.druid.segment.RowBasedStorageAdapter;
 import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.timeline.SegmentId;
 import org.hamcrest.CoreMatchers;
@@ -119,6 +121,18 @@ public class LookupSegmentTest
     Assert.assertNull(LOOKUP_SEGMENT.asQueryableIndex());
   }
 
+  @Test
+  public void test_asStorageAdapter_getRowSignature()
+  {
+    Assert.assertEquals(
+        RowSignature.builder()
+                    .add("k", ColumnType.STRING)
+                    .add("v", ColumnType.STRING)
+                    .build(),
+        LOOKUP_SEGMENT.asStorageAdapter().getRowSignature()
+    );
+  }
+
   @Test
   public void test_asStorageAdapter_getAvailableDimensions()
   {
diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
index 1f46534266f..b8c9721b5a0 100644
--- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
+++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
@@ -255,12 +255,12 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     Assert.assertEquals(6, index.getNumRows());
     Assert.assertEquals(
         ImmutableList.of(
-            ImmutableList.of(timestamp, "1", "2", 1L),
-            ImmutableList.of(timestamp, "1", "2", 1L),
-            ImmutableList.of(timestamp + 1, "1", "2", 1L),
-            ImmutableList.of(timestamp, "3", "4", 1L),
-            ImmutableList.of(timestamp, "3", "4", 1L),
-            ImmutableList.of(timestamp + 1, "3", "4", 1L)
+            ImmutableList.of("1", "2", timestamp, 1L),
+            ImmutableList.of("1", "2", timestamp, 1L),
+            ImmutableList.of("1", "2", timestamp + 1, 1L),
+            ImmutableList.of("3", "4", timestamp, 1L),
+            ImmutableList.of("3", "4", timestamp, 1L),
+            ImmutableList.of("3", "4", timestamp + 1, 1L)
         ),
         FrameTestUtil.readRowsFromAdapter(new QueryableIndexStorageAdapter(index), null, false).toList()
     );
@@ -325,10 +325,10 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
     Assert.assertEquals(4, index.getNumRows());
     Assert.assertEquals(
         ImmutableList.of(
-            ImmutableList.of(timestamp, "1", "2", 2L),
-            ImmutableList.of(timestamp + 1, "1", "2", 1L),
-            ImmutableList.of(timestamp, "3", "4", 2L),
-            ImmutableList.of(timestamp + 1, "3", "4", 1L)
+            ImmutableList.of("1", "2", timestamp, 2L),
+            ImmutableList.of("1", "2", timestamp + 1, 1L),
+            ImmutableList.of("3", "4", timestamp, 2L),
+            ImmutableList.of("3", "4", timestamp + 1, 1L)
         ),
         FrameTestUtil.readRowsFromAdapter(new QueryableIndexStorageAdapter(index), null, false).toList()
     );
diff --git a/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
index b26da1ceba7..7ae04d9fcb1 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/HashJoinSegmentStorageAdapterTest.java
@@ -58,6 +58,31 @@ public class HashJoinSegmentStorageAdapterTest extends BaseHashJoinSegmentStorag
     );
   }
 
+  @Test
+  public void test_getRowSignature_factToCountry()
+  {
+    Assert.assertEquals(
+        ImmutableList.of(
+            "__time",
+            "channel",
+            "regionIsoCode",
+            "countryNumber",
+            "countryIsoCode",
+            "user",
+            "isRobot",
+            "isAnonymous",
+            "namespace",
+            "page",
+            "delta",
+            "channel_uniques",
+            "c1.countryNumber",
+            "c1.countryIsoCode",
+            "c1.countryName"
+        ),
+        Lists.newArrayList(makeFactToCountrySegment().getRowSignature().getColumnNames())
+    );
+  }
+
   @Test
   public void test_getAvailableDimensions_factToCountry()
   {
diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java
index 73daadc7dd1..9b8f7f4d916 100644
--- a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java
@@ -43,6 +43,7 @@ import org.apache.druid.segment.IndexMerger;
 import org.apache.druid.segment.IndexMergerV9;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.QueryableIndexSegment;
+import org.apache.druid.segment.QueryableIndexStorageAdapter;
 import org.apache.druid.segment.SegmentLazyLoadFailCallback;
 import org.apache.druid.segment.SimpleAscendingOffset;
 import org.apache.druid.segment.TestIndex;
@@ -141,9 +142,8 @@ public class BroadcastSegmentIndexedTableTest extends InitializedNullHandlingTes
         segment.getTotalSpace()
     );
     backingSegment = (QueryableIndexSegment) factory.factorize(dataSegment, segment, false, SegmentLazyLoadFailCallback.NOOP);
-
-    columnNames = ImmutableList.<String>builder().add(ColumnHolder.TIME_COLUMN_NAME)
-                                                 .addAll(backingSegment.asQueryableIndex().getColumnNames()).build();
+    columnNames =
+        new QueryableIndexStorageAdapter(backingSegment.asQueryableIndex()).getRowSignature().getColumnNames();
     broadcastTable = new BroadcastSegmentIndexedTable(backingSegment, keyColumns, dataSegment.getVersion());
   }
 
diff --git a/server/src/main/java/org/apache/druid/segment/realtime/sink/Sink.java b/server/src/main/java/org/apache/druid/segment/realtime/sink/Sink.java
index 8d4fc0dd7f8..5e59c0e9b15 100644
--- a/server/src/main/java/org/apache/druid/segment/realtime/sink/Sink.java
+++ b/server/src/main/java/org/apache/druid/segment/realtime/sink/Sink.java
@@ -24,6 +24,7 @@ import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
+import com.google.errorprone.annotations.concurrent.GuardedBy;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
@@ -37,7 +38,6 @@ import org.apache.druid.segment.ReferenceCountingSegment;
 import org.apache.druid.segment.SegmentReference;
 import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.column.ColumnFormat;
-import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.incremental.AppendableIndexSpec;
@@ -92,6 +92,7 @@ public class Sink implements Iterable<FireHydrant>, Overshadowable<Sink>
   private final LinkedHashSet<String> dimOrder = new LinkedHashSet<>();
 
   // columns excluding current index (the in-memory fire hydrant), includes __time column
+  @GuardedBy("hydrantLock")
   private final LinkedHashSet<String> columnsExcludingCurrIndex = new LinkedHashSet<>();
 
   // column types for columns in {@code columnsExcludingCurrIndex}
@@ -397,6 +398,7 @@ public class Sink implements Iterable<FireHydrant>, Overshadowable<Sink>
   /**
    * Merge the column from the index with the existing columns.
    */
+  @GuardedBy("hydrantLock")
   private void overwriteIndexDimensions(StorageAdapter storageAdapter)
   {
     RowSignature rowSignature = storageAdapter.getRowSignature();
@@ -414,20 +416,19 @@ public class Sink implements Iterable<FireHydrant>, Overshadowable<Sink>
     synchronized (hydrantLock) {
       RowSignature.Builder builder = RowSignature.builder();
 
-      builder.addTimeColumn();
-
+      // Add columns from columnsExcludingCurrIndex.
       for (String dim : columnsExcludingCurrIndex) {
-        if (!ColumnHolder.TIME_COLUMN_NAME.equals(dim)) {
-          builder.add(dim, columnTypeExcludingCurrIndex.get(dim));
-        }
+        builder.add(dim, columnTypeExcludingCurrIndex.get(dim));
       }
 
-      IncrementalIndexStorageAdapter incrementalIndexStorageAdapter = new IncrementalIndexStorageAdapter(currHydrant.getIndex());
-      RowSignature incrementalIndexSignature = incrementalIndexStorageAdapter.getRowSignature();
+      // Add columns from the currHydrant that do not yet exist in columnsExcludingCurrIndex.
+      IncrementalIndexStorageAdapter currStorageAdapter =
+          new IncrementalIndexStorageAdapter(currHydrant.getIndex());
+      RowSignature currSignature = currStorageAdapter.getRowSignature();
 
-      for (String dim : incrementalIndexSignature.getColumnNames()) {
-        if (!columnsExcludingCurrIndex.contains(dim) && !ColumnHolder.TIME_COLUMN_NAME.equals(dim)) {
-          builder.add(dim, incrementalIndexSignature.getColumnType(dim).orElse(null));
+      for (String dim : currSignature.getColumnNames()) {
+        if (!columnsExcludingCurrIndex.contains(dim)) {
+          builder.add(dim, currSignature.getColumnType(dim).orElse(null));
         }
       }
 

From 16517e348e6d43168c9a9020d1b83a650620b364 Mon Sep 17 00:00:00 2001
From: Adarsh Sanjeev <adarshsanjeev@gmail.com>
Date: Tue, 27 Aug 2024 10:47:57 +0530
Subject: [PATCH 56/99] Suppress CVE (#16851)

---
 owasp-dependency-check-suppressions.xml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/owasp-dependency-check-suppressions.xml b/owasp-dependency-check-suppressions.xml
index 41d71fbb24a..d25148eb0de 100644
--- a/owasp-dependency-check-suppressions.xml
+++ b/owasp-dependency-check-suppressions.xml
@@ -692,4 +692,11 @@
     <cve>CVE-2023-50386</cve>
     <cve>CVE-2023-50292</cve>
   </suppress>
+  <suppress>
+    <!-- Transitive dependency from Apache Hadoop Common, which does not have a fix that can be upgraded to yet -->
+    <notes><![CDATA[
+    file name: dnsjava-2.1.7.jar
+    ]]></notes>
+    <vulnerabilityName>CVE-2024-25638</vulnerabilityName>
+  </suppress>
 </suppressions>

From 3b88b57d7097d6fe8771e25a90bfaedccddb3c42 Mon Sep 17 00:00:00 2001
From: Adarsh Sanjeev <adarshsanjeev@gmail.com>
Date: Tue, 27 Aug 2024 11:35:48 +0530
Subject: [PATCH 57/99]  Add configurable final stages to MSQ ingestion queries
 (#16699)

* Add a segmentMorphFactory to MSQ.

* Add test

* Make argument nullable

* Fix Guice issues

* Merge with master

* Remove extra information

* Fix tests

* Create a utils class

* Refactor segment generation

* Fix javadoc

* Refactor

* Refactor

* Fix injection
---
 .../apache/druid/msq/exec/ControllerImpl.java | 391 +-----------------
 .../apache/druid/msq/guice/MSQSqlModule.java  |   2 +
 .../guice/MSQTerminalStageSpecFactory.java    |  37 ++
 .../msq/indexing/MSQCompactionRunner.java     |   3 +-
 .../destination/DataSourceMSQDestination.java |  23 +-
 .../SegmentGenerationStageSpec.java           | 115 ++++++
 .../destination/SegmentGenerationUtils.java   | 374 +++++++++++++++++
 .../destination/TerminalStageSpec.java        |  39 ++
 .../druid/msq/sql/MSQTaskQueryMaker.java      |  12 +-
 .../druid/msq/sql/MSQTaskSqlEngine.java       |  19 +-
 .../util/PassthroughAggregatorFactory.java    |   2 +-
 .../msq/exec/MSQParseExceptionsTest.java      |   4 +-
 .../apache/druid/msq/exec/MSQReplaceTest.java |   3 -
 .../msq/indexing/MSQCompactionRunnerTest.java |   6 +-
 .../msq/indexing/MSQControllerTaskTest.java   |   1 +
 .../DataSourceMSQDestinationTest.java         |  17 +
 .../resources/SqlStatementResourceTest.java   |   1 +
 .../apache/druid/msq/test/MSQTestBase.java    |   5 +-
 .../MSQTestDelegateDataSegmentPusher.java     |   2 +-
 .../util/SqlStatementResourceHelperTest.java  |   1 +
 20 files changed, 668 insertions(+), 389 deletions(-)
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQTerminalStageSpecFactory.java
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationStageSpec.java
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/TerminalStageSpec.java

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
index 875212605b8..46a24611351 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
@@ -29,7 +29,6 @@ import com.google.common.collect.Iterators;
 import com.google.common.collect.Ordering;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.MoreExecutors;
-import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import it.unimi.dsi.fastutil.ints.IntArraySet;
@@ -38,10 +37,7 @@ import it.unimi.dsi.fastutil.ints.IntSet;
 import org.apache.druid.client.indexing.ClientCompactionTaskTransformSpec;
 import org.apache.druid.common.guava.FutureUtils;
 import org.apache.druid.data.input.StringTuple;
-import org.apache.druid.data.input.impl.DimensionSchema;
 import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.data.input.impl.LongDimensionSchema;
-import org.apache.druid.data.input.impl.TimestampSpec;
 import org.apache.druid.discovery.BrokerClient;
 import org.apache.druid.error.DruidException;
 import org.apache.druid.frame.allocation.ArenaMemoryAllocator;
@@ -82,7 +78,6 @@ import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Either;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.JodaUtils;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.StringUtils;
@@ -102,6 +97,8 @@ import org.apache.druid.msq.indexing.WorkerCount;
 import org.apache.druid.msq.indexing.client.ControllerChatHandler;
 import org.apache.druid.msq.indexing.destination.DataSourceMSQDestination;
 import org.apache.druid.msq.indexing.destination.ExportMSQDestination;
+import org.apache.druid.msq.indexing.destination.SegmentGenerationStageSpec;
+import org.apache.druid.msq.indexing.destination.TerminalStageSpec;
 import org.apache.druid.msq.indexing.error.CanceledFault;
 import org.apache.druid.msq.indexing.error.CannotParseExternalDataFault;
 import org.apache.druid.msq.indexing.error.FaultsExceededChecker;
@@ -139,8 +136,6 @@ import org.apache.druid.msq.input.inline.InlineInputSpecSlicer;
 import org.apache.druid.msq.input.lookup.LookupInputSpec;
 import org.apache.druid.msq.input.lookup.LookupInputSpecSlicer;
 import org.apache.druid.msq.input.stage.InputChannels;
-import org.apache.druid.msq.input.stage.ReadablePartition;
-import org.apache.druid.msq.input.stage.StageInputSlice;
 import org.apache.druid.msq.input.stage.StageInputSpec;
 import org.apache.druid.msq.input.stage.StageInputSpecSlicer;
 import org.apache.druid.msq.input.table.TableInputSpec;
@@ -166,28 +161,21 @@ import org.apache.druid.msq.querykit.scan.ScanQueryKit;
 import org.apache.druid.msq.shuffle.input.DurableStorageInputChannelFactory;
 import org.apache.druid.msq.shuffle.input.WorkerInputChannelFactory;
 import org.apache.druid.msq.statistics.PartialKeyStatisticsInformation;
-import org.apache.druid.msq.util.ArrayIngestMode;
-import org.apache.druid.msq.util.DimensionSchemaUtils;
 import org.apache.druid.msq.util.IntervalUtils;
 import org.apache.druid.msq.util.MSQFutureUtils;
 import org.apache.druid.msq.util.MSQTaskQueryMakerUtils;
 import org.apache.druid.msq.util.MultiStageQueryContext;
-import org.apache.druid.msq.util.PassthroughAggregatorFactory;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryContext;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.groupby.GroupByQuery;
-import org.apache.druid.query.groupby.GroupByQueryConfig;
 import org.apache.druid.query.operator.WindowOperatorQuery;
 import org.apache.druid.query.scan.ScanQuery;
-import org.apache.druid.segment.DimensionHandlerUtils;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
-import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.indexing.DataSchema;
-import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec;
 import org.apache.druid.segment.indexing.granularity.GranularitySpec;
 import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
 import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
@@ -195,7 +183,6 @@ import org.apache.druid.segment.transform.TransformSpec;
 import org.apache.druid.server.DruidNode;
 import org.apache.druid.sql.calcite.parser.DruidSqlInsert;
 import org.apache.druid.sql.calcite.planner.ColumnMappings;
-import org.apache.druid.sql.calcite.rel.DruidQuery;
 import org.apache.druid.sql.http.ResultFormat;
 import org.apache.druid.storage.ExportStorageProvider;
 import org.apache.druid.timeline.CompactionState;
@@ -205,7 +192,6 @@ import org.apache.druid.timeline.partition.NumberedPartialShardSpec;
 import org.apache.druid.timeline.partition.NumberedShardSpec;
 import org.apache.druid.timeline.partition.ShardSpec;
 import org.apache.druid.utils.CloseableUtils;
-import org.apache.druid.utils.CollectionUtils;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -218,7 +204,6 @@ import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -1192,11 +1177,16 @@ public class ControllerImpl implements Controller
   {
     if (MSQControllerTask.isIngestion(querySpec) &&
         stageNumber == queryDef.getFinalStageDefinition().getStageNumber()) {
-      // noinspection unchecked,rawtypes
-      return (Int2ObjectMap) makeSegmentGeneratorWorkerFactoryInfos(workerInputs, segmentsToGenerate);
-    } else {
-      return null;
+      final DataSourceMSQDestination destination = (DataSourceMSQDestination) querySpec.getDestination();
+      TerminalStageSpec terminalStageSpec = destination.getTerminalStageSpec();
+      if (destination.getTerminalStageSpec() instanceof SegmentGenerationStageSpec) {
+        return (Int2ObjectMap) ((SegmentGenerationStageSpec) terminalStageSpec).getWorkerInfo(
+            workerInputs,
+            segmentsToGenerate
+        );
+      }
     }
+    return null;
   }
 
   @SuppressWarnings("rawtypes")
@@ -1212,35 +1202,6 @@ public class ControllerImpl implements Controller
     return new MultiQueryKit(kitMap);
   }
 
-  private Int2ObjectMap<List<SegmentIdWithShardSpec>> makeSegmentGeneratorWorkerFactoryInfos(
-      final WorkerInputs workerInputs,
-      final List<SegmentIdWithShardSpec> segmentsToGenerate
-  )
-  {
-    final Int2ObjectMap<List<SegmentIdWithShardSpec>> retVal = new Int2ObjectAVLTreeMap<>();
-
-    // Empty segments validation already happens when the stages are started -- so we cannot have both
-    // isFailOnEmptyInsertEnabled and segmentsToGenerate.isEmpty() be true here.
-    if (segmentsToGenerate.isEmpty()) {
-      return retVal;
-    }
-
-    for (final int workerNumber : workerInputs.workers()) {
-      // SegmentGenerator stage has a single input from another stage.
-      final StageInputSlice stageInputSlice =
-          (StageInputSlice) Iterables.getOnlyElement(workerInputs.inputsForWorker(workerNumber));
-
-      final List<SegmentIdWithShardSpec> workerSegments = new ArrayList<>();
-      retVal.put(workerNumber, workerSegments);
-
-      for (final ReadablePartition partition : stageInputSlice.getPartitions()) {
-        workerSegments.add(segmentsToGenerate.get(partition.getPartitionNumber()));
-      }
-    }
-
-    return retVal;
-  }
-
   /**
    * A blocking function used to contact multiple workers. Checks if all the workers are running before contacting them.
    *
@@ -1816,9 +1777,6 @@ public class ControllerImpl implements Controller
     }
 
     if (MSQControllerTask.isIngestion(querySpec)) {
-      final RowSignature querySignature = queryDef.getFinalStageDefinition().getSignature();
-      final ClusterBy queryClusterBy = queryDef.getFinalStageDefinition().getClusterBy();
-
       // Find the stage that provides shuffled input to the final segment-generation stage.
       StageDefinition finalShuffleStageDef = queryDef.getFinalStageDefinition();
 
@@ -1845,24 +1803,15 @@ public class ControllerImpl implements Controller
         }
       }
 
-      // Then, add a segment-generation stage.
-      final DataSchema dataSchema =
-          makeDataSchemaForIngestion(querySpec, querySignature, queryClusterBy, columnMappings, jsonMapper);
-
-      builder.add(
-          StageDefinition.builder(queryDef.getNextStageNumber())
-                         .inputs(new StageInputSpec(queryDef.getFinalStageDefinition().getStageNumber()))
-                         .maxWorkerCount(tuningConfig.getMaxNumWorkers())
-                         .processorFactory(
-                             new SegmentGeneratorFrameProcessorFactory(
-                                 dataSchema,
-                                 columnMappings,
-                                 tuningConfig
-                             )
-                         )
-      );
-
-      return builder.build();
+      final DataSourceMSQDestination destination = (DataSourceMSQDestination) querySpec.getDestination();
+      return builder.add(
+          destination.getTerminalStageSpec()
+                     .constructFinalStage(
+                         queryDef,
+                         querySpec,
+                         jsonMapper)
+                    )
+                    .build();
     } else if (MSQControllerTask.writeFinalResultsToTaskReport(querySpec)) {
       return queryDef;
     } else if (MSQControllerTask.writeFinalStageResultsToDurableStorage(querySpec)) {
@@ -1933,112 +1882,6 @@ public class ControllerImpl implements Controller
     return ((DataSourceMSQDestination) querySpec.getDestination()).getDataSource();
   }
 
-  private static DataSchema makeDataSchemaForIngestion(
-      MSQSpec querySpec,
-      RowSignature querySignature,
-      ClusterBy queryClusterBy,
-      ColumnMappings columnMappings,
-      ObjectMapper jsonMapper
-  )
-  {
-    final DataSourceMSQDestination destination = (DataSourceMSQDestination) querySpec.getDestination();
-    final boolean isRollupQuery = isRollupQuery(querySpec.getQuery());
-    final boolean forceSegmentSortByTime =
-        MultiStageQueryContext.isForceSegmentSortByTime(querySpec.getQuery().context());
-
-    final Pair<DimensionsSpec, List<AggregatorFactory>> dimensionsAndAggregators =
-        makeDimensionsAndAggregatorsForIngestion(
-            querySignature,
-            queryClusterBy,
-            destination.getSegmentSortOrder(),
-            forceSegmentSortByTime,
-            columnMappings,
-            isRollupQuery,
-            querySpec.getQuery(),
-            destination.getDimensionSchemas()
-        );
-
-    return new DataSchema(
-        destination.getDataSource(),
-        new TimestampSpec(ColumnHolder.TIME_COLUMN_NAME, "millis", null),
-        dimensionsAndAggregators.lhs,
-        dimensionsAndAggregators.rhs.toArray(new AggregatorFactory[0]),
-        makeGranularitySpecForIngestion(querySpec.getQuery(), querySpec.getColumnMappings(), isRollupQuery, jsonMapper),
-        new TransformSpec(null, Collections.emptyList())
-    );
-  }
-
-  private static GranularitySpec makeGranularitySpecForIngestion(
-      final Query<?> query,
-      final ColumnMappings columnMappings,
-      final boolean isRollupQuery,
-      final ObjectMapper jsonMapper
-  )
-  {
-    if (isRollupQuery) {
-      final String queryGranularityString =
-          query.context().getString(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_GRANULARITY, "");
-
-      if (timeIsGroupByDimension((GroupByQuery) query, columnMappings) && !queryGranularityString.isEmpty()) {
-        final Granularity queryGranularity;
-
-        try {
-          queryGranularity = jsonMapper.readValue(queryGranularityString, Granularity.class);
-        }
-        catch (JsonProcessingException e) {
-          throw new RuntimeException(e);
-        }
-
-        return new ArbitraryGranularitySpec(queryGranularity, true, Intervals.ONLY_ETERNITY);
-      }
-      return new ArbitraryGranularitySpec(Granularities.NONE, true, Intervals.ONLY_ETERNITY);
-    } else {
-      return new ArbitraryGranularitySpec(Granularities.NONE, false, Intervals.ONLY_ETERNITY);
-    }
-  }
-
-  /**
-   * Checks that a {@link GroupByQuery} is grouping on the primary time column.
-   * <p>
-   * The logic here is roundabout. First, we check which column in the {@link GroupByQuery} corresponds to the
-   * output column {@link ColumnHolder#TIME_COLUMN_NAME}, using our {@link ColumnMappings}. Then, we check for the
-   * presence of an optimization done in {@link DruidQuery#toGroupByQuery()}, where the context parameter
-   * {@link GroupByQuery#CTX_TIMESTAMP_RESULT_FIELD} and various related parameters are set when one of the dimensions
-   * is detected to be a time-floor. Finally, we check that the name of that dimension, and the name of our time field
-   * from {@link ColumnMappings}, are the same.
-   */
-  private static boolean timeIsGroupByDimension(GroupByQuery groupByQuery, ColumnMappings columnMappings)
-  {
-    final IntList positions = columnMappings.getOutputColumnsByName(ColumnHolder.TIME_COLUMN_NAME);
-
-    if (positions.size() == 1) {
-      final String queryTimeColumn = columnMappings.getQueryColumnName(positions.getInt(0));
-      return queryTimeColumn.equals(groupByQuery.context().getString(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD));
-    } else {
-      return false;
-    }
-  }
-
-  /**
-   * Whether a native query represents an ingestion with rollup.
-   * <p>
-   * Checks for three things:
-   * <p>
-   * - The query must be a {@link GroupByQuery}, because rollup requires columns to be split into dimensions and
-   * aggregations.
-   * - The query must not finalize aggregations, because rollup requires inserting the intermediate type of
-   * complex aggregations, not the finalized type. (So further rollup is possible.)
-   * - The query must explicitly disable {@link GroupByQueryConfig#CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING}, because
-   * groupBy on multi-value dimensions implicitly unnests, which is not desired behavior for rollup at ingestion time
-   * (rollup expects multi-value dimensions to be treated as arrays).
-   */
-  private static boolean isRollupQuery(Query<?> query)
-  {
-    return query instanceof GroupByQuery
-           && !MultiStageQueryContext.isFinalizeAggregations(query.context())
-           && !query.context().getBoolean(GroupByQueryConfig.CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING, true);
-  }
-
   /**
    * Compute shard columns for {@link DimensionRangeShardSpec}. Returns an empty list if range-based sharding
    * is not applicable.
@@ -2147,197 +1990,6 @@ public class ControllerImpl implements Controller
     return new StringTuple(array);
   }
 
-  private static DimensionSchema getDimensionSchema(
-      final String outputColumnName,
-      @Nullable final ColumnType queryType,
-      QueryContext context,
-      @Nullable Map<String, DimensionSchema> dimensionSchemas
-  )
-  {
-    if (dimensionSchemas != null && dimensionSchemas.containsKey(outputColumnName)) {
-      return dimensionSchemas.get(outputColumnName);
-    }
-    // In case of ingestion, or when metrics are converted to dimensions when compaction is performed without rollu
-
-    // we won't have an entry in the map. For those cases, use the default config.
-    return DimensionSchemaUtils.createDimensionSchema(
-        outputColumnName,
-        queryType,
-        MultiStageQueryContext.useAutoColumnSchemas(context),
-        MultiStageQueryContext.getArrayIngestMode(context)
-    );
-  }
-
-  private static Pair<DimensionsSpec, List<AggregatorFactory>> makeDimensionsAndAggregatorsForIngestion(
-      final RowSignature querySignature,
-      final ClusterBy queryClusterBy,
-      final List<String> contextSegmentSortOrder,
-      final boolean forceSegmentSortByTime,
-      final ColumnMappings columnMappings,
-      final boolean isRollupQuery,
-      final Query<?> query,
-      @Nullable final Map<String, DimensionSchema> dimensionSchemas
-  )
-  {
-    // Log a warning unconditionally if arrayIngestMode is MVD, since the behaviour is incorrect, and is subject to
-    // deprecation and removal in future
-    if (MultiStageQueryContext.getArrayIngestMode(query.context()) == ArrayIngestMode.MVD) {
-      log.warn(
-          "%s[mvd] is active for this task. This causes string arrays (VARCHAR ARRAY in SQL) to be ingested as "
-          + "multi-value strings rather than true arrays. This behavior may change in a future version of Druid. To be "
-          + "compatible with future behavior changes, we recommend setting %s to[array], which creates a clearer "
-          + "separation between multi-value strings and true arrays. In either[mvd] or[array] mode, you can write "
-          + "out multi-value string dimensions using ARRAY_TO_MV. "
-          + "See https://druid.apache.org/docs/latest/querying/arrays#arrayingestmode for more details.",
-          MultiStageQueryContext.CTX_ARRAY_INGEST_MODE,
-          MultiStageQueryContext.CTX_ARRAY_INGEST_MODE
-      );
-    }
-
-    final List<DimensionSchema> dimensions = new ArrayList<>();
-    final List<AggregatorFactory> aggregators = new ArrayList<>();
-
-    // During ingestion, segment sort order is determined by the order of fields in the DimensionsSchema. We want
-    // this to match user intent as dictated by the declared segment sort order and CLUSTERED BY, so add things in
-    // that order.
-
-    // Start with segmentSortOrder.
-    final Set<String> outputColumnsInOrder = new LinkedHashSet<>(contextSegmentSortOrder);
-
-    // Then __time, if it's an output column and forceSegmentSortByTime is set.
-    if (columnMappings.hasOutputColumn(ColumnHolder.TIME_COLUMN_NAME) && forceSegmentSortByTime) {
-      outputColumnsInOrder.add(ColumnHolder.TIME_COLUMN_NAME);
-    }
-
-    // Then the query-level CLUSTERED BY.
-    // Note: this doesn't work when CLUSTERED BY specifies an expression that is not being selected.
-    // Such fields in CLUSTERED BY still control partitioning as expected, but do not affect sort order of rows
-    // within an individual segment.
-    for (final KeyColumn clusterByColumn : queryClusterBy.getColumns()) {
-      final IntList outputColumns = columnMappings.getOutputColumnsForQueryColumn(clusterByColumn.columnName());
-      for (final int outputColumn : outputColumns) {
-        outputColumnsInOrder.add(columnMappings.getOutputColumnName(outputColumn));
-      }
-    }
-
-    // Then all other columns.
-    outputColumnsInOrder.addAll(columnMappings.getOutputColumnNames());
-
-    Map<String, AggregatorFactory> outputColumnAggregatorFactories = new HashMap<>();
-
-    if (isRollupQuery) {
-      // Populate aggregators from the native query when doing an ingest in rollup mode.
-      for (AggregatorFactory aggregatorFactory : ((GroupByQuery) query).getAggregatorSpecs()) {
-        for (final int outputColumn : columnMappings.getOutputColumnsForQueryColumn(aggregatorFactory.getName())) {
-          final String outputColumnName = columnMappings.getOutputColumnName(outputColumn);
-          if (outputColumnAggregatorFactories.containsKey(outputColumnName)) {
-            throw new ISE("There can only be one aggregation for column [%s].", outputColumn);
-          } else {
-            outputColumnAggregatorFactories.put(
-                outputColumnName,
-                aggregatorFactory.withName(outputColumnName).getCombiningFactory()
-            );
-          }
-        }
-      }
-    }
-
-    // Each column can be either a dimension or an aggregator.
-    // For non-complex columns, If the aggregator factory of the column is not available, we treat the column as
-    // a dimension. For complex columns, certains hacks are in place.
-    for (final String outputColumnName : outputColumnsInOrder) {
-      // CollectionUtils.getOnlyElement because this method is only called during ingestion, where we require
-      // that output names be unique.
-      final int outputColumn = CollectionUtils.getOnlyElement(
-          columnMappings.getOutputColumnsByName(outputColumnName),
-          xs -> new ISE("Expected single output column for name [%s], but got [%s]", outputColumnName, xs)
-      );
-      final String queryColumn = columnMappings.getQueryColumnName(outputColumn);
-      final ColumnType type =
-          querySignature.getColumnType(queryColumn)
-                        .orElseThrow(() -> new ISE("No type for column [%s]", outputColumnName));
-
-      if (!type.is(ValueType.COMPLEX)) {
-        // non complex columns
-        populateDimensionsAndAggregators(
-            dimensions,
-            aggregators,
-            outputColumnAggregatorFactories,
-            outputColumnName,
-            type,
-            query.context(),
-            dimensionSchemas
-        );
-      } else {
-        // complex columns only
-        if (DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.containsKey(type.getComplexTypeName())) {
-          dimensions.add(
-              getDimensionSchema(outputColumnName, type, query.context(), dimensionSchemas)
-          );
-        } else if (!isRollupQuery) {
-          aggregators.add(new PassthroughAggregatorFactory(outputColumnName, type.getComplexTypeName()));
-        } else {
-          populateDimensionsAndAggregators(
-              dimensions,
-              aggregators,
-              outputColumnAggregatorFactories,
-              outputColumnName,
-              type,
-              query.context(),
-              dimensionSchemas
-          );
-        }
-      }
-    }
-
-    final DimensionsSpec.Builder dimensionsSpecBuilder = DimensionsSpec.builder();
-
-    if (!dimensions.isEmpty() && dimensions.get(0).getName().equals(ColumnHolder.TIME_COLUMN_NAME)) {
-      // Skip __time if it's in the first position, for compatibility with legacy dimensionSpecs.
-      dimensions.remove(0);
-      dimensionsSpecBuilder.setForceSegmentSortByTime(null);
-    } else {
-      // Store explicit forceSegmentSortByTime only if false, for compatibility with legacy dimensionSpecs.
-      dimensionsSpecBuilder.setForceSegmentSortByTime(forceSegmentSortByTime ? null : false);
-    }
-
-    return Pair.of(dimensionsSpecBuilder.setDimensions(dimensions).build(), aggregators);
-  }
-
-  /**
-   * If the output column is present in the outputColumnAggregatorFactories that means we already have the aggregator information for this column.
-   * else treat this column as a dimension.
-   *
-   * @param dimensions                      list is poulated if the output col is deemed to be a dimension
-   * @param aggregators                     list is populated with the aggregator if the output col is deemed to be a aggregation column.
-   * @param outputColumnAggregatorFactories output col -> AggregatorFactory map
-   * @param outputColumn                    column name
-   * @param type                            columnType
-   */
-  private static void populateDimensionsAndAggregators(
-      List<DimensionSchema> dimensions,
-      List<AggregatorFactory> aggregators,
-      Map<String, AggregatorFactory> outputColumnAggregatorFactories,
-      String outputColumn,
-      ColumnType type,
-      QueryContext context,
-      Map<String, DimensionSchema> dimensionSchemas
-  )
-  {
-    if (ColumnHolder.TIME_COLUMN_NAME.equals(outputColumn)) {
-      if (!type.is(ValueType.LONG)) {
-        throw DruidException.defensive("Incorrect type[%s] for column[%s]", type, outputColumn);
-      }
-      dimensions.add(new LongDimensionSchema(outputColumn));
-    } else if (outputColumnAggregatorFactories.containsKey(outputColumn)) {
-      aggregators.add(outputColumnAggregatorFactories.get(outputColumn));
-    } else {
-      dimensions.add(
-          getDimensionSchema(outputColumn, type, context, dimensionSchemas)
-      );
-    }
-  }
-
   private static DateTime getBucketDateTime(
       final ClusterByPartition partitionBoundary,
       final Granularity segmentGranularity,
@@ -2815,7 +2467,8 @@ public class ControllerImpl implements Controller
         for (final StageId stageId : newStageIds) {
           // Allocate segments, if this is the final stage of an ingestion.
           if (MSQControllerTask.isIngestion(querySpec)
-              && stageId.getStageNumber() == queryDef.getFinalStageDefinition().getStageNumber()) {
+              && stageId.getStageNumber() == queryDef.getFinalStageDefinition().getStageNumber()
+              && (((DataSourceMSQDestination) querySpec.getDestination()).getTerminalStageSpec() instanceof SegmentGenerationStageSpec)) {
             populateSegmentsToGenerate();
           }
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQSqlModule.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQSqlModule.java
index ea6eb364cec..a523fd6c012 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQSqlModule.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQSqlModule.java
@@ -54,6 +54,8 @@ public class MSQSqlModule implements DruidModule
     // We want this module to bring InputSourceModule along for the ride.
     binder.install(new InputSourceModule());
 
+    binder.bind(MSQTerminalStageSpecFactory.class).toInstance(new MSQTerminalStageSpecFactory());
+
     binder.bind(MSQTaskSqlEngine.class).in(LazySingleton.class);
 
     // Set up the EXTERN macro.
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQTerminalStageSpecFactory.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQTerminalStageSpecFactory.java
new file mode 100644
index 00000000000..d347322a7fb
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQTerminalStageSpecFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.guice;
+
+import org.apache.druid.msq.indexing.destination.SegmentGenerationStageSpec;
+import org.apache.druid.msq.indexing.destination.TerminalStageSpec;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.rel.DruidQuery;
+
+public class MSQTerminalStageSpecFactory
+{
+  /**
+   * Creates a {@link TerminalStageSpec} which determines the final of a query. Currently, always returns a segment
+   * generation spec, but this can be used to configure a wide range of behaviours.
+   */
+  public TerminalStageSpec createTerminalStageSpec(DruidQuery druidQuery, PlannerContext plannerContext)
+  {
+    return SegmentGenerationStageSpec.instance();
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
index fa011429763..5457e04286c 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
@@ -243,7 +243,8 @@ public class MSQCompactionRunner implements CompactionRunner
         dataSchema.getDimensionsSpec()
                   .getDimensions()
                   .stream()
-                  .collect(Collectors.toMap(DimensionSchema::getName, Function.identity()))
+                  .collect(Collectors.toMap(DimensionSchema::getName, Function.identity())),
+        null
     );
   }
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
index 6276c588d9d..485bceb2d09 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestination.java
@@ -51,6 +51,8 @@ public class DataSourceMSQDestination implements MSQDestination
   @Nullable
   private final List<Interval> replaceTimeChunks;
 
+  private final TerminalStageSpec terminalStageSpec;
+
   @Nullable
   private final Map<String, DimensionSchema> dimensionSchemas;
 
@@ -60,7 +62,8 @@ public class DataSourceMSQDestination implements MSQDestination
       @JsonProperty("segmentGranularity") Granularity segmentGranularity,
       @JsonProperty("segmentSortOrder") @Nullable List<String> segmentSortOrder,
       @JsonProperty("replaceTimeChunks") @Nullable List<Interval> replaceTimeChunks,
-      @JsonProperty("dimensionSchemas") @Nullable Map<String, DimensionSchema> dimensionSchemas
+      @JsonProperty("dimensionSchemas") @Nullable Map<String, DimensionSchema> dimensionSchemas,
+      @JsonProperty("terminalStageSpec") @Nullable TerminalStageSpec terminalStageSpec
   )
   {
     this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource");
@@ -68,6 +71,7 @@ public class DataSourceMSQDestination implements MSQDestination
     this.segmentSortOrder = segmentSortOrder != null ? segmentSortOrder : Collections.emptyList();
     this.replaceTimeChunks = replaceTimeChunks;
     this.dimensionSchemas = dimensionSchemas;
+    this.terminalStageSpec = terminalStageSpec != null ? terminalStageSpec : SegmentGenerationStageSpec.instance();
 
     if (replaceTimeChunks != null) {
       // Verify that if replaceTimeChunks is provided, it is nonempty.
@@ -105,6 +109,17 @@ public class DataSourceMSQDestination implements MSQDestination
     return dataSource;
   }
 
+  /**
+   * Returns the terminal stage spec.
+   * <p>
+   * The terminal stage spec, is a way to tell the MSQ task how to convert the results into segments at the final stage.
+   */
+  @JsonProperty
+  public TerminalStageSpec getTerminalStageSpec()
+  {
+    return terminalStageSpec;
+  }
+
   @JsonProperty
   public Granularity getSegmentGranularity()
   {
@@ -177,13 +192,14 @@ public class DataSourceMSQDestination implements MSQDestination
            && Objects.equals(segmentGranularity, that.segmentGranularity)
            && Objects.equals(segmentSortOrder, that.segmentSortOrder)
            && Objects.equals(replaceTimeChunks, that.replaceTimeChunks)
-           && Objects.equals(dimensionSchemas, that.dimensionSchemas);
+           && Objects.equals(dimensionSchemas, that.dimensionSchemas)
+           && Objects.equals(terminalStageSpec, that.terminalStageSpec);
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(dataSource, segmentGranularity, segmentSortOrder, replaceTimeChunks, dimensionSchemas);
+    return Objects.hash(dataSource, segmentGranularity, segmentSortOrder, replaceTimeChunks, dimensionSchemas, terminalStageSpec);
   }
 
   @Override
@@ -195,6 +211,7 @@ public class DataSourceMSQDestination implements MSQDestination
            ", segmentSortOrder=" + segmentSortOrder +
            ", replaceTimeChunks=" + replaceTimeChunks +
            ", dimensionSchemas=" + dimensionSchemas +
+           ", terminalStageSpec=" + terminalStageSpec +
            '}';
   }
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationStageSpec.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationStageSpec.java
new file mode 100644
index 00000000000..131d926288b
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationStageSpec.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.indexing.destination;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Iterables;
+import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import org.apache.druid.frame.key.ClusterBy;
+import org.apache.druid.msq.indexing.MSQSpec;
+import org.apache.druid.msq.indexing.MSQTuningConfig;
+import org.apache.druid.msq.indexing.processor.SegmentGeneratorFrameProcessorFactory;
+import org.apache.druid.msq.input.stage.ReadablePartition;
+import org.apache.druid.msq.input.stage.StageInputSlice;
+import org.apache.druid.msq.input.stage.StageInputSpec;
+import org.apache.druid.msq.kernel.QueryDefinition;
+import org.apache.druid.msq.kernel.StageDefinition;
+import org.apache.druid.msq.kernel.StageDefinitionBuilder;
+import org.apache.druid.msq.kernel.controller.WorkerInputs;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.indexing.DataSchema;
+import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
+import org.apache.druid.sql.calcite.planner.ColumnMappings;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SegmentGenerationStageSpec implements TerminalStageSpec
+{
+  public static final String TYPE = "segmentGeneration";
+
+  private static final SegmentGenerationStageSpec INSTANCE = new SegmentGenerationStageSpec();
+
+  private SegmentGenerationStageSpec()
+  {
+  }
+
+  @JsonCreator
+  public static SegmentGenerationStageSpec instance()
+  {
+    return INSTANCE;
+  }
+
+  @Override
+  public StageDefinitionBuilder constructFinalStage(QueryDefinition queryDef, MSQSpec querySpec, ObjectMapper jsonMapper)
+  {
+    final MSQTuningConfig tuningConfig = querySpec.getTuningConfig();
+    final ColumnMappings columnMappings = querySpec.getColumnMappings();
+    final RowSignature querySignature = queryDef.getFinalStageDefinition().getSignature();
+    final ClusterBy queryClusterBy = queryDef.getFinalStageDefinition().getClusterBy();
+
+    // Add a segment-generation stage.
+    final DataSchema dataSchema =
+        SegmentGenerationUtils.makeDataSchemaForIngestion(querySpec, querySignature, queryClusterBy, columnMappings, jsonMapper);
+
+    return StageDefinition.builder(queryDef.getNextStageNumber())
+                       .inputs(new StageInputSpec(queryDef.getFinalStageDefinition().getStageNumber()))
+                       .maxWorkerCount(tuningConfig.getMaxNumWorkers())
+                       .processorFactory(
+                           new SegmentGeneratorFrameProcessorFactory(
+                               dataSchema,
+                               columnMappings,
+                               tuningConfig
+                           )
+    );
+  }
+
+  public Int2ObjectMap<List<SegmentIdWithShardSpec>> getWorkerInfo(
+      final WorkerInputs workerInputs,
+      @Nullable final List<SegmentIdWithShardSpec> segmentsToGenerate
+  )
+  {
+    final Int2ObjectMap<List<SegmentIdWithShardSpec>> retVal = new Int2ObjectAVLTreeMap<>();
+
+    // Empty segments validation already happens when the stages are started -- so we cannot have both
+    // isFailOnEmptyInsertEnabled and segmentsToGenerate.isEmpty() be true here.
+    if (segmentsToGenerate == null || segmentsToGenerate.isEmpty()) {
+      return retVal;
+    }
+
+    for (final int workerNumber : workerInputs.workers()) {
+      // SegmentGenerator stage has a single input from another stage.
+      final StageInputSlice stageInputSlice =
+          (StageInputSlice) Iterables.getOnlyElement(workerInputs.inputsForWorker(workerNumber));
+
+      final List<SegmentIdWithShardSpec> workerSegments = new ArrayList<>();
+      retVal.put(workerNumber, workerSegments);
+
+      for (final ReadablePartition partition : stageInputSlice.getPartitions()) {
+        workerSegments.add(segmentsToGenerate.get(partition.getPartitionNumber()));
+      }
+    }
+
+    return retVal;
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java
new file mode 100644
index 00000000000..09d79534337
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.indexing.destination;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import it.unimi.dsi.fastutil.ints.IntList;
+import org.apache.druid.data.input.impl.DimensionSchema;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.LongDimensionSchema;
+import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.frame.key.ClusterBy;
+import org.apache.druid.frame.key.KeyColumn;
+import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.msq.indexing.MSQSpec;
+import org.apache.druid.msq.util.ArrayIngestMode;
+import org.apache.druid.msq.util.DimensionSchemaUtils;
+import org.apache.druid.msq.util.MultiStageQueryContext;
+import org.apache.druid.msq.util.PassthroughAggregatorFactory;
+import org.apache.druid.query.Query;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.query.groupby.GroupByQuery;
+import org.apache.druid.query.groupby.GroupByQueryConfig;
+import org.apache.druid.segment.DimensionHandlerUtils;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.column.ValueType;
+import org.apache.druid.segment.indexing.DataSchema;
+import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec;
+import org.apache.druid.segment.indexing.granularity.GranularitySpec;
+import org.apache.druid.segment.transform.TransformSpec;
+import org.apache.druid.sql.calcite.planner.ColumnMappings;
+import org.apache.druid.sql.calcite.rel.DruidQuery;
+import org.apache.druid.utils.CollectionUtils;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public final class SegmentGenerationUtils
+{
+  private static final Logger log = new Logger(SegmentGenerationUtils.class);
+
+  public static DataSchema makeDataSchemaForIngestion(
+      MSQSpec querySpec,
+      RowSignature querySignature,
+      ClusterBy queryClusterBy,
+      ColumnMappings columnMappings,
+      ObjectMapper jsonMapper
+  )
+  {
+    final DataSourceMSQDestination destination = (DataSourceMSQDestination) querySpec.getDestination();
+    final boolean isRollupQuery = isRollupQuery(querySpec.getQuery());
+    final boolean forceSegmentSortByTime =
+        MultiStageQueryContext.isForceSegmentSortByTime(querySpec.getQuery().context());
+
+    final Pair<DimensionsSpec, List<AggregatorFactory>> dimensionsAndAggregators =
+        makeDimensionsAndAggregatorsForIngestion(
+            querySignature,
+            queryClusterBy,
+            destination.getSegmentSortOrder(),
+            forceSegmentSortByTime,
+            columnMappings,
+            isRollupQuery,
+            querySpec.getQuery(),
+            destination.getDimensionSchemas()
+        );
+
+    return new DataSchema(
+        destination.getDataSource(),
+        new TimestampSpec(ColumnHolder.TIME_COLUMN_NAME, "millis", null),
+        dimensionsAndAggregators.lhs,
+        dimensionsAndAggregators.rhs.toArray(new AggregatorFactory[0]),
+        makeGranularitySpecForIngestion(querySpec.getQuery(), querySpec.getColumnMappings(), isRollupQuery, jsonMapper),
+        new TransformSpec(null, Collections.emptyList())
+    );
+  }
+
+  private static GranularitySpec makeGranularitySpecForIngestion(
+      final Query<?> query,
+      final ColumnMappings columnMappings,
+      final boolean isRollupQuery,
+      final ObjectMapper jsonMapper
+  )
+  {
+    if (isRollupQuery) {
+      final String queryGranularityString =
+          query.context().getString(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_GRANULARITY, "");
+
+      if (timeIsGroupByDimension((GroupByQuery) query, columnMappings) && !queryGranularityString.isEmpty()) {
+        final Granularity queryGranularity;
+
+        try {
+          queryGranularity = jsonMapper.readValue(queryGranularityString, Granularity.class);
+        }
+        catch (JsonProcessingException e) {
+          throw new RuntimeException(e);
+        }
+
+        return new ArbitraryGranularitySpec(queryGranularity, true, Intervals.ONLY_ETERNITY);
+      }
+      return new ArbitraryGranularitySpec(Granularities.NONE, true, Intervals.ONLY_ETERNITY);
+    } else {
+      return new ArbitraryGranularitySpec(Granularities.NONE, false, Intervals.ONLY_ETERNITY);
+    }
+  }
+
+  /**
+   * Checks that a {@link GroupByQuery} is grouping on the primary time column.
+   * <p>
+   * The logic here is roundabout. First, we check which column in the {@link GroupByQuery} corresponds to the
+   * output column {@link ColumnHolder#TIME_COLUMN_NAME}, using our {@link ColumnMappings}. Then, we check for the
+   * presence of an optimization done in {@link DruidQuery#toGroupByQuery()}, where the context parameter
+   * {@link GroupByQuery#CTX_TIMESTAMP_RESULT_FIELD} and various related parameters are set when one of the dimensions
+   * is detected to be a time-floor. Finally, we check that the name of that dimension, and the name of our time field
+   * from {@link ColumnMappings}, are the same.
+   */
+  private static boolean timeIsGroupByDimension(GroupByQuery groupByQuery, ColumnMappings columnMappings)
+  {
+    final IntList positions = columnMappings.getOutputColumnsByName(ColumnHolder.TIME_COLUMN_NAME);
+
+    if (positions.size() == 1) {
+      final String queryTimeColumn = columnMappings.getQueryColumnName(positions.getInt(0));
+      return queryTimeColumn.equals(groupByQuery.context().getString(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD));
+    } else {
+      return false;
+    }
+  }
+
+  /**
+   * Whether a native query represents an ingestion with rollup.
+   * <p>
+   * Checks for three things:
+   * <p>
+   * - The query must be a {@link GroupByQuery}, because rollup requires columns to be split into dimensions and
+   * aggregations.
+   * - The query must not finalize aggregations, because rollup requires inserting the intermediate type of
+   * complex aggregations, not the finalized type. (So further rollup is possible.)
+   * - The query must explicitly disable {@link GroupByQueryConfig#CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING}, because
+   * groupBy on multi-value dimensions implicitly unnests, which is not desired behavior for rollup at ingestion time
+   * (rollup expects multi-value dimensions to be treated as arrays).
+   */
+  private static boolean isRollupQuery(Query<?> query)
+  {
+    return query instanceof GroupByQuery
+           && !MultiStageQueryContext.isFinalizeAggregations(query.context())
+           && !query.context().getBoolean(GroupByQueryConfig.CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING, true);
+  }
+
+  private static DimensionSchema getDimensionSchema(
+      final String outputColumnName,
+      @Nullable final ColumnType queryType,
+      QueryContext context,
+      @Nullable Map<String, DimensionSchema> dimensionSchemas
+  )
+  {
+    if (dimensionSchemas != null && dimensionSchemas.containsKey(outputColumnName)) {
+      return dimensionSchemas.get(outputColumnName);
+    }
+    // In case of ingestion, or when metrics are converted to dimensions when compaction is performed without rollu
+
+    // we won't have an entry in the map. For those cases, use the default config.
+    return DimensionSchemaUtils.createDimensionSchema(
+        outputColumnName,
+        queryType,
+        MultiStageQueryContext.useAutoColumnSchemas(context),
+        MultiStageQueryContext.getArrayIngestMode(context)
+    );
+  }
+
+  private static Pair<DimensionsSpec, List<AggregatorFactory>> makeDimensionsAndAggregatorsForIngestion(
+      final RowSignature querySignature,
+      final ClusterBy queryClusterBy,
+      final List<String> contextSegmentSortOrder,
+      final boolean forceSegmentSortByTime,
+      final ColumnMappings columnMappings,
+      final boolean isRollupQuery,
+      final Query<?> query,
+      @Nullable final Map<String, DimensionSchema> dimensionSchemas
+  )
+  {
+    // Log a warning unconditionally if arrayIngestMode is MVD, since the behaviour is incorrect, and is subject to
+    // deprecation and removal in future
+    if (MultiStageQueryContext.getArrayIngestMode(query.context()) == ArrayIngestMode.MVD) {
+      log.warn(
+          "%s[mvd] is active for this task. This causes string arrays (VARCHAR ARRAY in SQL) to be ingested as "
+          + "multi-value strings rather than true arrays. This behavior may change in a future version of Druid. To be "
+          + "compatible with future behavior changes, we recommend setting %s to[array], which creates a clearer "
+          + "separation between multi-value strings and true arrays. In either[mvd] or[array] mode, you can write "
+          + "out multi-value string dimensions using ARRAY_TO_MV. "
+          + "See https://druid.apache.org/docs/latest/querying/arrays#arrayingestmode for more details.",
+          MultiStageQueryContext.CTX_ARRAY_INGEST_MODE,
+          MultiStageQueryContext.CTX_ARRAY_INGEST_MODE
+      );
+    }
+
+    final List<DimensionSchema> dimensions = new ArrayList<>();
+    final List<AggregatorFactory> aggregators = new ArrayList<>();
+
+    // During ingestion, segment sort order is determined by the order of fields in the DimensionsSchema. We want
+    // this to match user intent as dictated by the declared segment sort order and CLUSTERED BY, so add things in
+    // that order.
+
+    // Start with segmentSortOrder.
+    final Set<String> outputColumnsInOrder = new LinkedHashSet<>(contextSegmentSortOrder);
+
+    // Then __time, if it's an output column and forceSegmentSortByTime is set.
+    if (columnMappings.hasOutputColumn(ColumnHolder.TIME_COLUMN_NAME) && forceSegmentSortByTime) {
+      outputColumnsInOrder.add(ColumnHolder.TIME_COLUMN_NAME);
+    }
+
+    // Then the query-level CLUSTERED BY.
+    // Note: this doesn't work when CLUSTERED BY specifies an expression that is not being selected.
+    // Such fields in CLUSTERED BY still control partitioning as expected, but do not affect sort order of rows
+    // within an individual segment.
+    for (final KeyColumn clusterByColumn : queryClusterBy.getColumns()) {
+      final IntList outputColumns = columnMappings.getOutputColumnsForQueryColumn(clusterByColumn.columnName());
+      for (final int outputColumn : outputColumns) {
+        outputColumnsInOrder.add(columnMappings.getOutputColumnName(outputColumn));
+      }
+    }
+
+    // Then all other columns.
+    outputColumnsInOrder.addAll(columnMappings.getOutputColumnNames());
+
+    Map<String, AggregatorFactory> outputColumnAggregatorFactories = new HashMap<>();
+
+    if (isRollupQuery) {
+      // Populate aggregators from the native query when doing an ingest in rollup mode.
+      for (AggregatorFactory aggregatorFactory : ((GroupByQuery) query).getAggregatorSpecs()) {
+        for (final int outputColumn : columnMappings.getOutputColumnsForQueryColumn(aggregatorFactory.getName())) {
+          final String outputColumnName = columnMappings.getOutputColumnName(outputColumn);
+          if (outputColumnAggregatorFactories.containsKey(outputColumnName)) {
+            throw new ISE("There can only be one aggregation for column [%s].", outputColumn);
+          } else {
+            outputColumnAggregatorFactories.put(
+                outputColumnName,
+                aggregatorFactory.withName(outputColumnName).getCombiningFactory()
+            );
+          }
+        }
+      }
+    }
+
+    // Each column can be either a dimension or an aggregator.
+    // For non-complex columns, If the aggregator factory of the column is not available, we treat the column as
+    // a dimension. For complex columns, certains hacks are in place.
+    for (final String outputColumnName : outputColumnsInOrder) {
+      // CollectionUtils.getOnlyElement because this method is only called during ingestion, where we require
+      // that output names be unique.
+      final int outputColumn = CollectionUtils.getOnlyElement(
+          columnMappings.getOutputColumnsByName(outputColumnName),
+          xs -> new ISE("Expected single output column for name [%s], but got [%s]", outputColumnName, xs)
+      );
+      final String queryColumn = columnMappings.getQueryColumnName(outputColumn);
+      final ColumnType type =
+          querySignature.getColumnType(queryColumn)
+                        .orElseThrow(() -> new ISE("No type for column [%s]", outputColumnName));
+
+      if (!type.is(ValueType.COMPLEX)) {
+        // non complex columns
+        populateDimensionsAndAggregators(
+            dimensions,
+            aggregators,
+            outputColumnAggregatorFactories,
+            outputColumnName,
+            type,
+            query.context(),
+            dimensionSchemas
+        );
+      } else {
+        // complex columns only
+        if (DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.containsKey(type.getComplexTypeName())) {
+          dimensions.add(
+              getDimensionSchema(outputColumnName, type, query.context(), dimensionSchemas)
+          );
+        } else if (!isRollupQuery) {
+          aggregators.add(new PassthroughAggregatorFactory(outputColumnName, type.getComplexTypeName()));
+        } else {
+          populateDimensionsAndAggregators(
+              dimensions,
+              aggregators,
+              outputColumnAggregatorFactories,
+              outputColumnName,
+              type,
+              query.context(),
+              dimensionSchemas
+          );
+        }
+      }
+    }
+
+    final DimensionsSpec.Builder dimensionsSpecBuilder = DimensionsSpec.builder();
+
+    if (!dimensions.isEmpty() && dimensions.get(0).getName().equals(ColumnHolder.TIME_COLUMN_NAME)) {
+      // Skip __time if it's in the first position, for compatibility with legacy dimensionSpecs.
+      dimensions.remove(0);
+      dimensionsSpecBuilder.setForceSegmentSortByTime(null);
+    } else {
+      // Store explicit forceSegmentSortByTime only if false, for compatibility with legacy dimensionSpecs.
+      dimensionsSpecBuilder.setForceSegmentSortByTime(forceSegmentSortByTime ? null : false);
+    }
+
+    return Pair.of(dimensionsSpecBuilder.setDimensions(dimensions).build(), aggregators);
+  }
+
+  /**
+   * If the output column is present in the outputColumnAggregatorFactories that means we already have the aggregator information for this column.
+   * else treat this column as a dimension.
+   *
+   * @param dimensions                      list is poulated if the output col is deemed to be a dimension
+   * @param aggregators                     list is populated with the aggregator if the output col is deemed to be a aggregation column.
+   * @param outputColumnAggregatorFactories output col -> AggregatorFactory map
+   * @param outputColumn                    column name
+   * @param type                            columnType
+   */
+  private static void populateDimensionsAndAggregators(
+      List<DimensionSchema> dimensions,
+      List<AggregatorFactory> aggregators,
+      Map<String, AggregatorFactory> outputColumnAggregatorFactories,
+      String outputColumn,
+      ColumnType type,
+      QueryContext context,
+      Map<String, DimensionSchema> dimensionSchemas
+  )
+  {
+    if (ColumnHolder.TIME_COLUMN_NAME.equals(outputColumn)) {
+      if (!type.is(ValueType.LONG)) {
+        throw DruidException.defensive("Incorrect type[%s] for column[%s]", type, outputColumn);
+      }
+      dimensions.add(new LongDimensionSchema(outputColumn));
+    } else if (outputColumnAggregatorFactories.containsKey(outputColumn)) {
+      aggregators.add(outputColumnAggregatorFactories.get(outputColumn));
+    } else {
+      dimensions.add(
+          getDimensionSchema(outputColumn, type, context, dimensionSchemas)
+      );
+    }
+  }
+
+  private SegmentGenerationUtils()
+  {
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/TerminalStageSpec.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/TerminalStageSpec.java
new file mode 100644
index 00000000000..6bae67954fd
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/TerminalStageSpec.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.indexing.destination;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.druid.msq.indexing.MSQSpec;
+import org.apache.druid.msq.kernel.QueryDefinition;
+import org.apache.druid.msq.kernel.StageDefinitionBuilder;
+
+/**
+ * Determines the final stage of a {@link DataSourceMSQDestination}.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonSubTypes(value = {
+    @JsonSubTypes.Type(name = SegmentGenerationStageSpec.TYPE, value = SegmentGenerationStageSpec.class)
+})
+public interface TerminalStageSpec
+{
+  StageDefinitionBuilder constructFinalStage(QueryDefinition queryDef, MSQSpec querySpec, ObjectMapper jsonMapper);
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
index 60fa197c4a5..830fb87e1b2 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java
@@ -33,6 +33,7 @@ import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.msq.exec.MSQTasks;
+import org.apache.druid.msq.guice.MSQTerminalStageSpecFactory;
 import org.apache.druid.msq.indexing.MSQControllerTask;
 import org.apache.druid.msq.indexing.MSQSpec;
 import org.apache.druid.msq.indexing.MSQTuningConfig;
@@ -92,6 +93,7 @@ public class MSQTaskQueryMaker implements QueryMaker
   private final PlannerContext plannerContext;
   private final ObjectMapper jsonMapper;
   private final List<Entry<Integer, String>> fieldMapping;
+  private final MSQTerminalStageSpecFactory terminalStageSpecFactory;
 
 
   MSQTaskQueryMaker(
@@ -99,7 +101,8 @@ public class MSQTaskQueryMaker implements QueryMaker
       final OverlordClient overlordClient,
       final PlannerContext plannerContext,
       final ObjectMapper jsonMapper,
-      final List<Entry<Integer, String>> fieldMapping
+      final List<Entry<Integer, String>> fieldMapping,
+      final MSQTerminalStageSpecFactory terminalStageSpecFactory
   )
   {
     this.targetDataSource = targetDataSource;
@@ -107,6 +110,7 @@ public class MSQTaskQueryMaker implements QueryMaker
     this.plannerContext = Preconditions.checkNotNull(plannerContext, "plannerContext");
     this.jsonMapper = Preconditions.checkNotNull(jsonMapper, "jsonMapper");
     this.fieldMapping = Preconditions.checkNotNull(fieldMapping, "fieldMapping");
+    this.terminalStageSpecFactory = terminalStageSpecFactory;
   }
 
   @Override
@@ -247,7 +251,11 @@ public class MSQTaskQueryMaker implements QueryMaker
           segmentGranularityObject,
           segmentSortOrder,
           replaceTimeChunks,
-          null
+          null,
+          terminalStageSpecFactory.createTerminalStageSpec(
+              druidQuery,
+              plannerContext
+          )
       );
       MultiStageQueryContext.validateAndGetTaskLockType(sqlQueryContext, dataSourceDestination.isReplaceTimeChunks());
       destination = dataSourceDestination;
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
index d20abffb0f9..24fbd373c95 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
@@ -20,6 +20,7 @@
 package org.apache.druid.msq.sql;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Inject;
@@ -42,6 +43,7 @@ import org.apache.druid.error.InvalidSqlInput;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.msq.guice.MSQTerminalStageSpecFactory;
 import org.apache.druid.msq.querykit.QueryKitUtils;
 import org.apache.druid.msq.util.ArrayIngestMode;
 import org.apache.druid.msq.util.DimensionSchemaUtils;
@@ -85,15 +87,24 @@ public class MSQTaskSqlEngine implements SqlEngine
 
   private final OverlordClient overlordClient;
   private final ObjectMapper jsonMapper;
+  private final MSQTerminalStageSpecFactory terminalStageSpecFactory;
 
   @Inject
+  public MSQTaskSqlEngine(final OverlordClient overlordClient, final ObjectMapper jsonMapper)
+  {
+    this(overlordClient, jsonMapper, new MSQTerminalStageSpecFactory());
+  }
+
+  @VisibleForTesting
   public MSQTaskSqlEngine(
       final OverlordClient overlordClient,
-      final ObjectMapper jsonMapper
+      final ObjectMapper jsonMapper,
+      final MSQTerminalStageSpecFactory terminalStageSpecFactory
   )
   {
     this.overlordClient = overlordClient;
     this.jsonMapper = jsonMapper;
+    this.terminalStageSpecFactory = terminalStageSpecFactory;
   }
 
   @Override
@@ -162,7 +173,8 @@ public class MSQTaskSqlEngine implements SqlEngine
         overlordClient,
         plannerContext,
         jsonMapper,
-        relRoot.fields
+        relRoot.fields,
+        terminalStageSpecFactory
     );
   }
 
@@ -195,7 +207,8 @@ public class MSQTaskSqlEngine implements SqlEngine
         overlordClient,
         plannerContext,
         jsonMapper,
-        relRoot.fields
+        relRoot.fields,
+        terminalStageSpecFactory
     );
   }
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/PassthroughAggregatorFactory.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/PassthroughAggregatorFactory.java
index 8de47936eb4..f821c169505 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/PassthroughAggregatorFactory.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/PassthroughAggregatorFactory.java
@@ -39,7 +39,7 @@ import java.util.Objects;
  * Hack that allows "passing through" arbitrary complex types into
  * {@link org.apache.druid.segment.incremental.IncrementalIndex}.
  *
- * Used by {@link org.apache.druid.msq.exec.ControllerImpl#makeDimensionsAndAggregatorsForIngestion}.
+ * Used by {@link org.apache.druid.msq.indexing.destination.SegmentGenerationUtils#makeDimensionsAndAggregatorsForIngestion}.
  *
  * To move away from this, it would need to be possible to create complex columns in segments only knowing the complex
  * type; in particular, without knowing the type of an aggregator factory or dimension schema that corresponds to
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java
index bc8d517ffba..879da23977e 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java
@@ -225,7 +225,7 @@ public class MSQParseExceptionsTest extends MSQTestBase
                         new ColumnMapping("v1", "agent_category")
                     )
                 ))
-                .destination(new DataSourceMSQDestination("foo1", Granularities.ALL, null, null, null))
+                .destination(new DataSourceMSQDestination("foo1", Granularities.ALL, null, null, null, null))
                 .tuningConfig(MSQTuningConfig.defaultConfig())
                 .build())
         .setQueryContext(DEFAULT_MSQ_CONTEXT)
@@ -318,7 +318,7 @@ public class MSQParseExceptionsTest extends MSQTestBase
                         new ColumnMapping("agent_category", "agent_category")
                     )
                 ))
-                .destination(new DataSourceMSQDestination("foo1", Granularities.ALL, null, null, null))
+                .destination(new DataSourceMSQDestination("foo1", Granularities.ALL, null, null, null, null))
                 .tuningConfig(MSQTuningConfig.defaultConfig())
                 .build())
         .setQueryContext(runtimeContext)
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
index e470a951877..72541046913 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
@@ -1375,7 +1375,6 @@ public class MSQReplaceTest extends MSQTestBase
                              + "WHERE __time >= TIMESTAMP '2000-01-01' AND __time < TIMESTAMP '2000-01-03' "
                              + "PARTITIONED BY MONTH")
                      .setExpectedDataSource("foo")
-                     .setQueryContext(DEFAULT_MSQ_CONTEXT)
                      .setExpectedRowSignature(rowSignature)
                      .setQueryContext(context)
                      .setExpectedDestinationIntervals(Collections.singletonList(Intervals.of("2000-01-01T/2002-01-01T")))
@@ -1432,7 +1431,6 @@ public class MSQReplaceTest extends MSQTestBase
                              + "WHERE __time >= TIMESTAMP '2000-01-01' AND __time < TIMESTAMP '2000-01-03' "
                              + "PARTITIONED BY MONTH")
                      .setExpectedDataSource("foo")
-                     .setQueryContext(DEFAULT_MSQ_CONTEXT)
                      .setExpectedRowSignature(rowSignature)
                      .setQueryContext(context)
                      .setExpectedDestinationIntervals(Collections.singletonList(Intervals.ETERNITY))
@@ -1480,7 +1478,6 @@ public class MSQReplaceTest extends MSQTestBase
                          "REPLACE INTO foo1 OVERWRITE ALL "
                          + "select  __time, dim1 , count(*) as cnt from foo  where dim1 is not null group by 1, 2 PARTITIONED by day clustered by dim1")
                      .setExpectedDataSource("foo1")
-                     .setQueryContext(DEFAULT_MSQ_CONTEXT)
                      .setExpectedShardSpec(DimensionRangeShardSpec.class)
                      .setExpectedMSQSegmentReport(
                          new MSQSegmentReport(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
index 94be3495b98..09c5ae47718 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
@@ -290,7 +290,8 @@ public class MSQCompactionRunnerTest
             SEGMENT_GRANULARITY.getDefaultGranularity(),
             null,
             Collections.singletonList(COMPACTION_INTERVAL),
-            DIMENSIONS.stream().collect(Collectors.toMap(DimensionSchema::getName, Function.identity()))
+            DIMENSIONS.stream().collect(Collectors.toMap(DimensionSchema::getName, Function.identity())),
+            null
         ),
         actualMSQSpec.getDestination()
     );
@@ -360,7 +361,8 @@ public class MSQCompactionRunnerTest
             SEGMENT_GRANULARITY.getDefaultGranularity(),
             null,
             Collections.singletonList(COMPACTION_INTERVAL),
-            DIMENSIONS.stream().collect(Collectors.toMap(DimensionSchema::getName, Function.identity()))
+            DIMENSIONS.stream().collect(Collectors.toMap(DimensionSchema::getName, Function.identity())),
+            null
         ),
         actualMSQSpec.getDestination()
     );
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java
index e969e209387..76586c1e108 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java
@@ -59,6 +59,7 @@ public class MSQControllerTaskTest
           Granularities.DAY,
           null,
           INTERVALS,
+          null,
           null
       ))
       .query(new Druids.ScanQueryBuilder()
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
index 331430181a9..6b843df9187 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java
@@ -20,10 +20,14 @@
 package org.apache.druid.msq.indexing.destination;
 
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.collect.ImmutableMap;
 import nl.jqno.equalsverifier.EqualsVerifier;
 import org.apache.druid.data.input.impl.DimensionSchema;
 import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.junit.Assert;
 import org.junit.Test;
 
 import java.util.Map;
@@ -58,4 +62,17 @@ public class DataSourceMSQDestinationTest
                   .usingGetClass()
                   .verify();
   }
+
+  @Test
+  public void testBackwardCompatibility() throws JsonProcessingException
+  {
+    DataSourceMSQDestination destination = new DataSourceMSQDestination("foo1", Granularities.ALL, null, null, null, null);
+    Assert.assertEquals(SegmentGenerationStageSpec.instance(), destination.getTerminalStageSpec());
+
+    DataSourceMSQDestination dataSourceMSQDestination = new DefaultObjectMapper().readValue(
+        "{\"type\":\"dataSource\",\"dataSource\":\"datasource1\",\"segmentGranularity\":\"DAY\",\"rowsInTaskReport\":0,\"destinationResource\":{\"empty\":false,\"present\":true}}",
+        DataSourceMSQDestination.class
+    );
+    Assert.assertEquals(SegmentGenerationStageSpec.instance(), dataSourceMSQDestination.getTerminalStageSpec());
+  }
 }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java
index 2a753e21d16..a79edee53b8 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java
@@ -200,6 +200,7 @@ public class SqlStatementResourceTest extends MSQTestBase
                  Granularities.DAY,
                  null,
                  null,
+                 null,
                  null
              ))
              .tuningConfig(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
index 258b5c97751..fe725528f40 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
@@ -91,6 +91,7 @@ import org.apache.druid.msq.guice.MSQDurableStorageModule;
 import org.apache.druid.msq.guice.MSQExternalDataSourceModule;
 import org.apache.druid.msq.guice.MSQIndexingModule;
 import org.apache.druid.msq.guice.MSQSqlModule;
+import org.apache.druid.msq.guice.MSQTerminalStageSpecFactory;
 import org.apache.druid.msq.guice.MultiStageQuery;
 import org.apache.druid.msq.indexing.InputChannelFactory;
 import org.apache.druid.msq.indexing.MSQControllerTask;
@@ -210,7 +211,6 @@ import org.mockito.Mockito;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -550,7 +550,8 @@ public class MSQTestBase extends BaseCalciteQueryTest
 
     final SqlEngine engine = new MSQTaskSqlEngine(
         indexingServiceClient,
-        qf.queryJsonMapper().copy().registerModules(new MSQSqlModule().getJacksonModules())
+        qf.queryJsonMapper().copy().registerModules(new MSQSqlModule().getJacksonModules()),
+        new MSQTerminalStageSpecFactory()
     );
 
     PlannerFactory plannerFactory = new PlannerFactory(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestDelegateDataSegmentPusher.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestDelegateDataSegmentPusher.java
index 61a4389e93d..73fca53682c 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestDelegateDataSegmentPusher.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestDelegateDataSegmentPusher.java
@@ -40,7 +40,7 @@ public class MSQTestDelegateDataSegmentPusher implements DataSegmentPusher
       MSQTestSegmentManager segmentManager
   )
   {
-    delegate = dataSegmentPusher;
+    this.delegate = dataSegmentPusher;
     this.segmentManager = segmentManager;
   }
 
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/SqlStatementResourceHelperTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/SqlStatementResourceHelperTest.java
index 58856adf366..7fb6bc687a8 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/SqlStatementResourceHelperTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/SqlStatementResourceHelperTest.java
@@ -376,6 +376,7 @@ public class SqlStatementResourceHelperTest
             Granularities.DAY,
             null,
             null,
+            null,
             null
         )
     );

From ed3dbd62424ef49fa69316e50c907e6891fbd3e8 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Tue, 27 Aug 2024 00:02:32 -0700
Subject: [PATCH 58/99] MSQ: Fix validation of time position in collations.
 (#16961)

* MSQ: Fix validation of time position in collations.

It is possible for the collation to refer to a field that isn't mapped,
such as when the DML includes "CLUSTERED BY some_function(some_field)".
In this case, the collation refers to a projected column that is not
part of the field mappings. Prior to this patch, that would lead to an
out of bounds list access on fieldMappings.

This patch fixes the problem by identifying the position of __time in
the fieldMappings first, rather than retrieving each collation field
from fieldMappings.

Fixes a bug introduced in #16849.

* Fix test. Better warning message.
---
 .../druid/msq/sql/MSQTaskSqlEngine.java       | 23 ++++--
 .../apache/druid/msq/exec/MSQReplaceTest.java | 80 +++++++++++++++++++
 .../druid/data/input/impl/DimensionsSpec.java |  6 +-
 3 files changed, 99 insertions(+), 10 deletions(-)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
index 24fbd373c95..01d87805f78 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
@@ -479,22 +479,29 @@ public class MSQTaskSqlEngine implements SqlEngine
         );
       }
     } else if (!rootCollation.getFieldCollations().isEmpty()) {
-      int timePosition = -1;
-      for (int i = 0; i < rootCollation.getFieldCollations().size(); i++) {
-        final String fieldCollationName =
-            fieldMappings.get(rootCollation.getFieldCollations().get(i).getFieldIndex()).getValue();
-        if (ColumnHolder.TIME_COLUMN_NAME.equals(fieldCollationName)) {
-          timePosition = i;
+      int timePositionInRow = -1;
+      for (int i = 0; i < fieldMappings.size(); i++) {
+        Entry<Integer, String> entry = fieldMappings.get(i);
+        if (ColumnHolder.TIME_COLUMN_NAME.equals(entry.getValue())) {
+          timePositionInRow = i;
           break;
         }
       }
 
-      if (timePosition > 0) {
+      int timePositionInCollation = -1;
+      for (int i = 0; i < rootCollation.getFieldCollations().size(); i++) {
+        if (rootCollation.getFieldCollations().get(i).getFieldIndex() == timePositionInRow) {
+          timePositionInCollation = i;
+          break;
+        }
+      }
+
+      if (timePositionInCollation > 0) {
         throw InvalidSqlInput.exception(
             "Sort order (CLUSTERED BY) cannot include[%s] in position[%d] unless context parameter[%s] "
             + "is set to[false]. %s",
             ColumnHolder.TIME_COLUMN_NAME,
-            timePosition,
+            timePositionInCollation,
             MultiStageQueryContext.CTX_FORCE_TIME_SORT,
             DimensionsSpec.WARNING_NON_TIME_SORT_ORDER
         );
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
index 72541046913..798a27a95e4 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
@@ -277,6 +277,86 @@ public class MSQReplaceTest extends MSQTestBase
   @MethodSource("data")
   @ParameterizedTest(name = "{index}:with context {0}")
   public void testReplaceOnFooWithAllClusteredByDimExplicitSort(String contextName, Map<String, Object> context)
+  {
+    // Tests [CLUSTERED BY LOWER(dim1)], i.e. an expression that is not actually stored.
+    RowSignature rowSignature = RowSignature.builder()
+                                            .add("__time", ColumnType.LONG)
+                                            .add("dim1", ColumnType.STRING)
+                                            .add("m1", ColumnType.FLOAT)
+                                            .build();
+
+    DataSegment existingDataSegment0 = DataSegment.builder()
+                                                  .interval(Intervals.of("2000-01-01T/2000-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    DataSegment existingDataSegment1 = DataSegment.builder()
+                                                  .interval(Intervals.of("2001-01-01T/2001-01-04T"))
+                                                  .size(50)
+                                                  .version(MSQTestTaskActionClient.VERSION)
+                                                  .dataSource("foo")
+                                                  .build();
+
+    Mockito.doCallRealMethod()
+           .doReturn(ImmutableSet.of(existingDataSegment0, existingDataSegment1))
+           .when(testTaskActionClient)
+           .submit(new RetrieveUsedSegmentsAction("foo", ImmutableList.of(Intervals.ETERNITY)));
+
+    testIngestQuery().setSql(" REPLACE INTO foo OVERWRITE ALL "
+                             + "SELECT __time, dim1, m1 "
+                             + "FROM foo "
+                             + "PARTITIONED BY ALL "
+                             + "CLUSTERED BY LOWER(dim1)")
+                     .setExpectedDataSource("foo")
+                     .setExpectedRowSignature(rowSignature)
+                     .setQueryContext(context)
+                     .setExpectedDestinationIntervals(Intervals.ONLY_ETERNITY)
+                     .setExpectedSegments(
+                         ImmutableSet.of(
+                             SegmentId.of("foo", Intervals.ETERNITY, "test", 0)
+                         )
+                     )
+                     .setExpectedShardSpec(NumberedShardSpec.class)
+                     .setExpectedResultRows(
+                         ImmutableList.of(
+                             new Object[]{946684800000L, NullHandling.sqlCompatible() ? "" : null, 1.0f},
+                             new Object[]{946771200000L, "10.1", 2.0f},
+                             new Object[]{946857600000L, "2", 3.0f},
+                             new Object[]{978307200000L, "1", 4.0f},
+                             new Object[]{978393600000L, "def", 5.0f},
+                             new Object[]{978480000000L, "abc", 6.0f}
+                         )
+                     )
+                     .setExpectedSegmentGenerationProgressCountersForStageWorker(
+                         CounterSnapshotMatcher
+                             .with().segmentRowsProcessed(6),
+                         1, 0
+                     )
+                     .setExpectedLastCompactionState(
+                         expectedCompactionState(
+                             context,
+                             Collections.emptyList(),
+                             DimensionsSpec.builder()
+                                           .setDimensions(
+                                               ImmutableList.of(
+                                                   new StringDimensionSchema("dim1"),
+                                                   new FloatDimensionSchema("m1")
+                                               )
+                                           )
+                                           .setDimensionExclusions(Collections.singletonList("__time"))
+                                           .build(),
+                             GranularityType.ALL,
+                             Intervals.ETERNITY
+                         )
+                     )
+                     .verifyResults();
+  }
+
+  @MethodSource("data")
+  @ParameterizedTest(name = "{index}:with context {0}")
+  public void testReplaceOnFooWithAllClusteredByExpression(String contextName, Map<String, Object> context)
   {
     RowSignature rowSignature = RowSignature.builder()
                                             .add("dim1", ColumnType.STRING)
diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionsSpec.java b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionsSpec.java
index e529aa2d218..efc3718d4ff 100644
--- a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionsSpec.java
+++ b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionsSpec.java
@@ -60,8 +60,10 @@ public class DimensionsSpec
   public static final String WARNING_NON_TIME_SORT_ORDER = StringUtils.format(
       "Warning: support for segments not sorted by[%s] is experimental. Such segments are not readable by older "
       + "version of Druid, and certain queries cannot run on them. See "
-      + "https://druid.apache.org/docs/latest/ingestion/partitioning#sorting for details before using this option.",
-      ColumnHolder.TIME_COLUMN_NAME
+      + "https://druid.apache.org/docs/latest/ingestion/partitioning#sorting for details before setting "
+      + "%s to[false].",
+      ColumnHolder.TIME_COLUMN_NAME,
+      PARAMETER_FORCE_TIME_SORT
   );
 
   public static final boolean DEFAULT_FORCE_TIME_SORT = true;

From f8301a314fdaa3e66eb357e632798f32e69b3c32 Mon Sep 17 00:00:00 2001
From: Clint Wylie <cwylie@apache.org>
Date: Tue, 27 Aug 2024 00:34:41 -0700
Subject: [PATCH 59/99] generic block compressed complex columns (#16863)

changes:
* Adds new `CompressedComplexColumn`, `CompressedComplexColumnSerializer`, `CompressedComplexColumnSupplier` based on `CompressedVariableSizedBlobColumn` used by JSON columns
* Adds `IndexSpec.complexMetricCompression` which can be used to specify compression for the generic compressed complex column. Defaults to uncompressed because compressed columns are not backwards compatible.
* Adds new definition of `ComplexMetricSerde.getSerializer` which accepts an `IndexSpec` argument when creating a serializer. The old signature has been marked `@Deprecated` and has a default implementation that returns `null`, but it will be used by the default implementation of the new version if it is implemented to return a non-null value. The default implementation of the new method will use a `CompressedComplexColumnSerializer` if `IndexSpec.complexMetricCompression` is not null/none/uncompressed, or will use `LargeColumnSupportedComplexColumnSerializer` otherwise.
* Removed all duplicate generic implementations of `ComplexMetricSerde.getSerializer` and `ComplexMetricSerde.deserializeColumn` into default implementations `ComplexMetricSerde` instead of being copied all over the place. The default implementation of `deserializeColumn` will check if the first byte indicates that the new compression was used, otherwise will use the `GenericIndexed` based supplier.
* Complex columns with custom serializers/deserializers are unaffected and may continue doing whatever it is they do, either with specialized compression or whatever else, this new stuff is just to provide generic implementations built around `ObjectStrategy`.
* add ObjectStrategy.readRetainsBufferReference so CompressedComplexColumn only copies on read if required
* add copyValueOnRead flag down to CompressedBlockReader to avoid buffer duplicate if the value needs copied anyway
---
 docs/ingestion/ingestion-spec.md              |   1 +
 .../CompressedBigDecimalColumn.java           |   4 +-
 .../CompressedBigDecimalMetricSerde.java      |   7 +-
 .../CompressedBigDecimalObjectStrategy.java   |   6 +
 .../ddsketch/DDSketchComplexMetricSerde.java  |  30 ----
 .../ddsketch/DDSketchObjectStrategy.java      |   6 +
 .../ddsketch/DDSketchObjectStrategyTest.java  |  33 ++++
 .../MomentSketchComplexMetricSerde.java       |  30 ----
 .../MomentSketchObjectStrategy.java           |   6 +
 .../MomentSketchObjectStrategyTest.java       |  33 ++++
 ...RabbitStreamIndexTaskTuningConfigTest.java | 139 ++++++++-------
 .../SpectatorHistogramComplexMetricSerde.java |   7 +-
 ...tatorHistogramIndexBasedComplexColumn.java |   3 +
 .../SpectatorHistogramObjectStrategy.java     |   6 +
 .../histogram/SpectatorHistogramTest.java     |   1 +
 .../TDigestSketchComplexMetricSerde.java      |  30 ----
 .../TDigestSketchObjectStrategy.java          |   6 +
 .../TDigestSketchObjectStrategyTest.java      |  33 ++++
 .../hll/HllSketchMergeComplexMetricSerde.java |  26 ---
 .../KllDoublesSketchComplexMetricSerde.java   |  22 ---
 .../KllFloatsSketchComplexMetricSerde.java    |  22 ---
 .../DoublesSketchComplexMetricSerde.java      |  22 ---
 .../theta/SketchMergeComplexMetricSerde.java  |  21 ---
 ...fDoublesSketchMergeComplexMetricSerde.java |  22 ---
 .../HllSketchHolderObjectStrategyTest.java    |   1 +
 ...llDoublesSketchComplexMetricSerdeTest.java |   1 +
 ...KllFloatsSketchComplexMetricSerdeTest.java |   1 +
 .../DoublesSketchComplexMetricSerdeTest.java  |   1 +
 .../theta/SketchHolderObjectStrategyTest.java |   1 +
 ...rrayOfDoublesSketchObjectStrategyTest.java |   1 +
 .../aggregation/bloom/BloomFilterSerde.java   |   7 +-
 .../ApproximateHistogramFoldingSerde.java     |  26 +--
 .../histogram/FixedBucketsHistogramSerde.java |  28 +--
 .../ApproximateHistogramFoldingSerdeTest.java |  10 ++
 .../histogram/FixedBucketsHistogramTest.java  |   9 +
 .../apache/druid/msq/exec/MSQReplaceTest.java |   2 +-
 .../aggregation/variance/VarianceSerde.java   |  26 +--
 .../variance/VarianceSerdeTest.java           |   1 +
 .../columnar/ComplexFrameColumnReader.java    |   1 +
 ...rializablePairLongDoubleComplexColumn.java |   2 +
 ...zablePairLongDoubleComplexMetricSerde.java |  13 +-
 ...erializablePairLongFloatComplexColumn.java |   2 +
 ...izablePairLongFloatComplexMetricSerde.java |  13 +-
 ...SerializablePairLongLongComplexColumn.java |   2 +
 ...lizablePairLongLongComplexMetricSerde.java |  13 +-
 ...rializablePairLongStringComplexColumn.java |   2 +
 ...zablePairLongStringComplexMetricSerde.java |  13 +-
 .../hyperloglog/HyperUniquesSerde.java        |  20 ---
 .../apache/druid/segment/IndexMergerV9.java   |   2 +-
 .../org/apache/druid/segment/IndexSpec.java   |  25 ++-
 .../druid/segment/column/ComplexColumn.java   |   2 +
 .../GenericIndexedBasedComplexColumn.java     |   3 +
 .../segment/column/NullableTypeStrategy.java  |  10 +-
 .../ObjectStrategyComplexTypeStrategy.java    |   3 +-
 .../druid/segment/column/TypeStrategy.java    |  10 +-
 .../column/UnknownTypeComplexColumn.java      |   2 +-
 .../segment/data/CompressedBlockReader.java   |  31 +++-
 .../segment/data/CompressedLongsReader.java   |   2 +-
 ...ressedVariableSizedBlobColumnSupplier.java |  20 ++-
 .../druid/segment/data/GenericIndexed.java    |   6 +
 .../druid/segment/data/ObjectStrategy.java    |  18 ++
 .../generator/GeneratorBasicSchemas.java      |  39 +++--
 .../CompressedNestedDataComplexColumn.java    |   2 +-
 .../segment/nested/DictionaryIdLookup.java    |  11 +-
 ...balDictionaryEncodedFieldColumnWriter.java |   3 +-
 .../NestedCommonFormatColumnSerializer.java   |  34 +---
 .../nested/NestedDataColumnSerializer.java    |  33 ++--
 .../nested/NestedDataColumnSerializerV4.java  |  10 +-
 .../nested/NestedDataColumnSupplier.java      |  13 +-
 .../nested/NestedDataColumnSupplierV4.java    |  22 +--
 .../segment/nested/NestedDataColumnV4.java    |   3 +-
 .../segment/nested/NestedDataColumnV5.java    |   3 +-
 .../nested/NestedDataComplexTypeSerde.java    |  30 ++--
 .../ScalarDoubleColumnAndIndexSupplier.java   |   7 +-
 .../nested/ScalarDoubleColumnSerializer.java  |   7 +-
 .../ScalarLongColumnAndIndexSupplier.java     |   7 +-
 .../nested/ScalarLongColumnSerializer.java    |   7 +-
 ...larNestedCommonFormatColumnSerializer.java |   5 +-
 .../ScalarStringColumnAndIndexSupplier.java   |   7 +-
 .../nested/ScalarStringColumnSerializer.java  |   3 +-
 .../druid/segment/nested/StructuredData.java  |   3 +-
 .../nested/VariantColumnAndIndexSupplier.java |  17 +-
 .../nested/VariantColumnSerializer.java       |  23 +--
 .../segment/serde/ColumnSerializerUtils.java  |  81 +++++++++
 .../ComplexColumnMetadata.java}               |   6 +-
 .../segment/serde/ComplexMetricSerde.java     | 124 ++++++++++----
 .../serde/CompressedComplexColumn.java        |  86 ++++++++++
 .../CompressedComplexColumnSerializer.java    | 161 ++++++++++++++++++
 .../CompressedComplexColumnSupplier.java      | 123 +++++++++++++
 .../NestedCommonFormatColumnPartSerde.java    |   3 +-
 .../druid/query/QueryRunnerTestHelper.java    |   4 +
 ...ePairLongDoubleComplexMetricSerdeTest.java |   3 +-
 ...lePairLongFloatComplexMetricSerdeTest.java |   3 +-
 ...blePairLongLongComplexMetricSerdeTest.java |   3 +-
 ...ePairLongStringComplexMetricSerdeTest.java |   5 +-
 .../org/apache/druid/segment/TestIndex.java   |  13 ++
 .../CompressedVariableSizeBlobColumnTest.java |   2 +-
 .../NestedDataColumnSupplierV4Test.java       |   3 +-
 .../serde/HyperUniquesSerdeForTest.java       |  13 --
 website/.spelling                             |   1 +
 100 files changed, 1155 insertions(+), 614 deletions(-)
 create mode 100644 extensions-contrib/ddsketch/src/test/java/org/apache/druid/query/aggregation/ddsketch/DDSketchObjectStrategyTest.java
 create mode 100644 extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchObjectStrategyTest.java
 create mode 100644 extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchObjectStrategyTest.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/serde/ColumnSerializerUtils.java
 rename processing/src/main/java/org/apache/druid/segment/{nested/NestedDataColumnMetadata.java => serde/ComplexColumnMetadata.java} (94%)
 create mode 100644 processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumn.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumnSerializer.java
 create mode 100644 processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumnSupplier.java

diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md
index f1e974a9cc2..a02bd276639 100644
--- a/docs/ingestion/ingestion-spec.md
+++ b/docs/ingestion/ingestion-spec.md
@@ -520,6 +520,7 @@ For information on defining an `indexSpec` in a query context, see [SQL-based in
 |stringDictionaryEncoding|Encoding format for string value dictionaries used by STRING and [COMPLEX&lt;json&gt;](../querying/nested-columns.md) columns. To enable front coding, set `stringDictionaryEncoding.type` to `frontCoded`. Optionally, you can specify the `bucketSize` and `formatVersion` properties. See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`|
 |metricCompression|Compression format for primitive type metric columns. Options are `lz4`, `lzf`, `zstd`, `uncompressed`, or `none` (which is more efficient than `uncompressed`, but not supported by older versions of Druid).|`lz4`|
 |longEncoding|Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. Options are `auto` or `longs`. `auto` encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. `longs` stores the value as-is with 8 bytes each.|`longs`|
+|complexMetricCompression|Compression format for complex type metric columns. Options are `lz4`, `lzf`, `zstd`, `uncompressed`. Options other than `uncompressed` are not compatible with Druid versions older than 31, and only applies to complex metrics which do not have specialized column formats.|`uncompressed`|
 |jsonCompression|Compression format to use for nested column raw data. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
 
 #### Front coding
diff --git a/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalColumn.java b/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalColumn.java
index a366675dc03..43901adfb97 100644
--- a/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalColumn.java
+++ b/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalColumn.java
@@ -69,6 +69,7 @@ public class CompressedBigDecimalColumn implements ComplexColumn
   }
 
   @Override
+  @Nullable
   public CompressedBigDecimal getRowValue(int rowNum)
   {
     int s = scale.get(rowNum);
@@ -96,7 +97,8 @@ public class CompressedBigDecimalColumn implements ComplexColumn
   {
     return new ObjectColumnSelector<CompressedBigDecimal>()
     {
-      @Override @Nullable
+      @Override
+      @Nullable
       public CompressedBigDecimal getObject()
       {
         return getRowValue(offset.getOffset());
diff --git a/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalMetricSerde.java b/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalMetricSerde.java
index 91f2f1a3484..1decd3df137 100644
--- a/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalMetricSerde.java
+++ b/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalMetricSerde.java
@@ -20,6 +20,7 @@
 package org.apache.druid.compressedbigdecimal;
 
 import org.apache.druid.data.input.InputRow;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.data.ObjectStrategy;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
@@ -73,7 +74,8 @@ public class CompressedBigDecimalMetricSerde extends ComplexMetricSerde
   public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder)
   {
     builder.setComplexColumnSupplier(
-        CompressedBigDecimalColumnPartSupplier.fromByteBuffer(buffer));
+        CompressedBigDecimalColumnPartSupplier.fromByteBuffer(buffer)
+    );
   }
 
   /* (non-Javadoc)
@@ -83,7 +85,8 @@ public class CompressedBigDecimalMetricSerde extends ComplexMetricSerde
   @Override
   public CompressedBigDecimalLongColumnSerializer getSerializer(
       SegmentWriteOutMedium segmentWriteOutMedium,
-      String column
+      String column,
+      IndexSpec indexSpec
   )
   {
     return CompressedBigDecimalLongColumnSerializer.create(segmentWriteOutMedium, column);
diff --git a/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalObjectStrategy.java b/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalObjectStrategy.java
index 829c1f94d58..bdf05cf3e3e 100644
--- a/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalObjectStrategy.java
+++ b/extensions-contrib/compressed-bigdecimal/src/main/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalObjectStrategy.java
@@ -91,4 +91,10 @@ public class CompressedBigDecimalObjectStrategy implements ObjectStrategy<Compre
 
     return buf.array();
   }
+
+  @Override
+  public boolean readRetainsBufferReference()
+  {
+    return false;
+  }
 }
diff --git a/extensions-contrib/ddsketch/src/main/java/org/apache/druid/query/aggregation/ddsketch/DDSketchComplexMetricSerde.java b/extensions-contrib/ddsketch/src/main/java/org/apache/druid/query/aggregation/ddsketch/DDSketchComplexMetricSerde.java
index f6820685b87..75cd29c3a37 100644
--- a/extensions-contrib/ddsketch/src/main/java/org/apache/druid/query/aggregation/ddsketch/DDSketchComplexMetricSerde.java
+++ b/extensions-contrib/ddsketch/src/main/java/org/apache/druid/query/aggregation/ddsketch/DDSketchComplexMetricSerde.java
@@ -22,17 +22,9 @@ package org.apache.druid.query.aggregation.ddsketch;
 import com.datadoghq.sketch.ddsketch.DDSketch;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
-
-import java.nio.ByteBuffer;
 
 
 public class DDSketchComplexMetricSerde extends ComplexMetricSerde
@@ -84,31 +76,9 @@ public class DDSketchComplexMetricSerde extends ComplexMetricSerde
     };
   }
 
-  @Override
-  public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder)
-  {
-    final GenericIndexed<DDSketch> column = GenericIndexed.read(
-        buffer,
-        STRATEGY,
-        builder.getFileMapper()
-    );
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
   @Override
   public ObjectStrategy<DDSketch> getObjectStrategy()
   {
     return STRATEGY;
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(
-        segmentWriteOutMedium,
-        column,
-        this.getObjectStrategy()
-    );
-  }
-
 }
diff --git a/extensions-contrib/ddsketch/src/main/java/org/apache/druid/query/aggregation/ddsketch/DDSketchObjectStrategy.java b/extensions-contrib/ddsketch/src/main/java/org/apache/druid/query/aggregation/ddsketch/DDSketchObjectStrategy.java
index fb0cc8044a0..219e23f817b 100644
--- a/extensions-contrib/ddsketch/src/main/java/org/apache/druid/query/aggregation/ddsketch/DDSketchObjectStrategy.java
+++ b/extensions-contrib/ddsketch/src/main/java/org/apache/druid/query/aggregation/ddsketch/DDSketchObjectStrategy.java
@@ -70,4 +70,10 @@ public class DDSketchObjectStrategy implements ObjectStrategy<DDSketch>
   {
     return DDSketchAggregatorFactory.COMPARATOR.compare(o1, o2);
   }
+
+  @Override
+  public boolean readRetainsBufferReference()
+  {
+    return false;
+  }
 }
diff --git a/extensions-contrib/ddsketch/src/test/java/org/apache/druid/query/aggregation/ddsketch/DDSketchObjectStrategyTest.java b/extensions-contrib/ddsketch/src/test/java/org/apache/druid/query/aggregation/ddsketch/DDSketchObjectStrategyTest.java
new file mode 100644
index 00000000000..03b52b08720
--- /dev/null
+++ b/extensions-contrib/ddsketch/src/test/java/org/apache/druid/query/aggregation/ddsketch/DDSketchObjectStrategyTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.aggregation.ddsketch;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class DDSketchObjectStrategyTest
+{
+  @Test
+  public void testReadRetainsBufferReference()
+  {
+    DDSketchObjectStrategy strategy = new DDSketchObjectStrategy();
+    Assert.assertFalse(strategy.readRetainsBufferReference());
+  }
+}
diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchComplexMetricSerde.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchComplexMetricSerde.java
index 4e631e53525..a0c3f1545da 100644
--- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchComplexMetricSerde.java
+++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchComplexMetricSerde.java
@@ -21,17 +21,9 @@ package org.apache.druid.query.aggregation.momentsketch;
 
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.query.aggregation.momentsketch.aggregator.MomentSketchAggregatorFactory;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
-
-import java.nio.ByteBuffer;
 
 public class MomentSketchComplexMetricSerde extends ComplexMetricSerde
 {
@@ -62,31 +54,9 @@ public class MomentSketchComplexMetricSerde extends ComplexMetricSerde
     };
   }
 
-  @Override
-  public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder)
-  {
-    final GenericIndexed<MomentSketchWrapper> column = GenericIndexed.read(
-        buffer,
-        STRATEGY,
-        builder.getFileMapper()
-    );
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
   @Override
   public ObjectStrategy<MomentSketchWrapper> getObjectStrategy()
   {
     return STRATEGY;
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(
-        segmentWriteOutMedium,
-        column,
-        this.getObjectStrategy()
-    );
-  }
-
 }
diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchObjectStrategy.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchObjectStrategy.java
index 7a706f90399..77fa9dc0d4e 100644
--- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchObjectStrategy.java
+++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchObjectStrategy.java
@@ -59,4 +59,10 @@ public class MomentSketchObjectStrategy implements ObjectStrategy<MomentSketchWr
   {
     return MomentSketchAggregatorFactory.COMPARATOR.compare(o1, o2);
   }
+
+  @Override
+  public boolean readRetainsBufferReference()
+  {
+    return false;
+  }
 }
diff --git a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchObjectStrategyTest.java b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchObjectStrategyTest.java
new file mode 100644
index 00000000000..5ef909dd8e7
--- /dev/null
+++ b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/MomentSketchObjectStrategyTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.aggregation.momentsketch;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MomentSketchObjectStrategyTest
+{
+  @Test
+  public void testReadRetainsBufferReference()
+  {
+    MomentSketchObjectStrategy strategy = new MomentSketchObjectStrategy();
+    Assert.assertFalse(strategy.readRetainsBufferReference());
+  }
+}
diff --git a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java
index 852942a8423..193b35b8af1 100644
--- a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java
+++ b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java
@@ -54,8 +54,10 @@ public class RabbitStreamIndexTaskTuningConfigTest
         mapper.writeValueAsString(
             mapper.readValue(
                 jsonStr,
-                TuningConfig.class)),
-        TuningConfig.class);
+                TuningConfig.class
+            )),
+        TuningConfig.class
+    );
 
     Assert.assertNull(config.getBasePersistDirectory());
     Assert.assertEquals(new OnheapIncrementalIndex.Spec(), config.getAppendableIndexSpec());
@@ -79,26 +81,28 @@ public class RabbitStreamIndexTaskTuningConfigTest
   public void testSerdeWithNonDefaults() throws Exception
   {
     String jsonStr = "{\n"
-        + "  \"type\": \"rabbit\",\n"
-        + "  \"basePersistDirectory\": \"/tmp/xxx\",\n"
-        + "  \"maxRowsInMemory\": 100,\n"
-        + "  \"maxRowsPerSegment\": 100,\n"
-        + "  \"intermediatePersistPeriod\": \"PT1H\",\n"
-        + "  \"maxPendingPersists\": 100,\n"
-        + "  \"reportParseExceptions\": true,\n"
-        + "  \"handoffConditionTimeout\": 100,\n"
-        + "  \"recordBufferSize\": 1000,\n"
-        + "  \"recordBufferOfferTimeout\": 500,\n"
-        + "  \"resetOffsetAutomatically\": false,\n"
-        + "  \"appendableIndexSpec\": { \"type\" : \"onheap\" }\n"
-        + "}";
+                     + "  \"type\": \"rabbit\",\n"
+                     + "  \"basePersistDirectory\": \"/tmp/xxx\",\n"
+                     + "  \"maxRowsInMemory\": 100,\n"
+                     + "  \"maxRowsPerSegment\": 100,\n"
+                     + "  \"intermediatePersistPeriod\": \"PT1H\",\n"
+                     + "  \"maxPendingPersists\": 100,\n"
+                     + "  \"reportParseExceptions\": true,\n"
+                     + "  \"handoffConditionTimeout\": 100,\n"
+                     + "  \"recordBufferSize\": 1000,\n"
+                     + "  \"recordBufferOfferTimeout\": 500,\n"
+                     + "  \"resetOffsetAutomatically\": false,\n"
+                     + "  \"appendableIndexSpec\": { \"type\" : \"onheap\" }\n"
+                     + "}";
 
     RabbitStreamIndexTaskTuningConfig config = (RabbitStreamIndexTaskTuningConfig) mapper.readValue(
         mapper.writeValueAsString(
             mapper.readValue(
                 jsonStr,
-                TuningConfig.class)),
-        TuningConfig.class);
+                TuningConfig.class
+            )),
+        TuningConfig.class
+    );
 
     Assert.assertNull(config.getBasePersistDirectory());
     Assert.assertEquals(new OnheapIncrementalIndex.Spec(), config.getAppendableIndexSpec());
@@ -119,62 +123,65 @@ public class RabbitStreamIndexTaskTuningConfigTest
   public void testtoString() throws Exception
   {
     String jsonStr = "{\n"
-        + "  \"type\": \"rabbit\",\n"
-        + "  \"basePersistDirectory\": \"/tmp/xxx\",\n"
-        + "  \"maxRowsInMemory\": 100,\n"
-        + "  \"maxRowsPerSegment\": 100,\n"
-        + "  \"intermediatePersistPeriod\": \"PT1H\",\n"
-        + "  \"maxPendingPersists\": 100,\n"
-        + "  \"reportParseExceptions\": true,\n"
-        + "  \"handoffConditionTimeout\": 100,\n"
-        + "  \"recordBufferSize\": 1000,\n"
-        + "  \"recordBufferOfferTimeout\": 500,\n"
-        + "  \"resetOffsetAutomatically\": false,\n"
-        + "  \"appendableIndexSpec\": { \"type\" : \"onheap\" }\n"
-        + "}";
+                     + "  \"type\": \"rabbit\",\n"
+                     + "  \"basePersistDirectory\": \"/tmp/xxx\",\n"
+                     + "  \"maxRowsInMemory\": 100,\n"
+                     + "  \"maxRowsPerSegment\": 100,\n"
+                     + "  \"intermediatePersistPeriod\": \"PT1H\",\n"
+                     + "  \"maxPendingPersists\": 100,\n"
+                     + "  \"reportParseExceptions\": true,\n"
+                     + "  \"handoffConditionTimeout\": 100,\n"
+                     + "  \"recordBufferSize\": 1000,\n"
+                     + "  \"recordBufferOfferTimeout\": 500,\n"
+                     + "  \"resetOffsetAutomatically\": false,\n"
+                     + "  \"appendableIndexSpec\": { \"type\" : \"onheap\" }\n"
+                     + "}";
 
     RabbitStreamIndexTaskTuningConfig config = (RabbitStreamIndexTaskTuningConfig) mapper.readValue(
         mapper.writeValueAsString(
             mapper.readValue(
                 jsonStr,
-                TuningConfig.class)),
-        TuningConfig.class);
+                TuningConfig.class
+            )),
+        TuningConfig.class
+    );
 
     String resStr = "RabbitStreamSupervisorTuningConfig{" +
-        "maxRowsInMemory=100, " +
-        "maxRowsPerSegment=100, " +
-        "maxTotalRows=null, " +
-        "maxBytesInMemory=" + config.getMaxBytesInMemoryOrDefault() + ", " +
-        "skipBytesInMemoryOverheadCheck=false, " +
-        "intermediatePersistPeriod=PT1H, " +
-        "maxPendingPersists=100, " +
-        "indexSpec=IndexSpec{" +
-        "bitmapSerdeFactory=RoaringBitmapSerdeFactory{}, " +
-        "dimensionCompression=lz4, " +
-        "stringDictionaryEncoding=Utf8{}, " +
-        "metricCompression=lz4, " +
-        "longEncoding=longs, " +
-        "jsonCompression=null, " +
-        "segmentLoader=null" +
-        "}, " +
-        "reportParseExceptions=true, " +
-        "handoffConditionTimeout=100, " +
-        "resetOffsetAutomatically=false, " +
-        "segmentWriteOutMediumFactory=null, " +
-        "workerThreads=null, " +
-        "chatRetries=8, " +
-        "httpTimeout=PT10S, " +
-        "shutdownTimeout=PT80S, " +
-        "recordBufferSize=1000, " +
-        "recordBufferOfferTimeout=500, " +
-        "offsetFetchPeriod=PT30S, " +
-        "intermediateHandoffPeriod=" + config.getIntermediateHandoffPeriod() + ", " +
-        "logParseExceptions=false, " +
-        "maxParseExceptions=0, " +
-        "maxSavedParseExceptions=0, " +
-        "numPersistThreads=1, " +
-        "maxRecordsPerPoll=null}";
-  
+                    "maxRowsInMemory=100, " +
+                    "maxRowsPerSegment=100, " +
+                    "maxTotalRows=null, " +
+                    "maxBytesInMemory=" + config.getMaxBytesInMemoryOrDefault() + ", " +
+                    "skipBytesInMemoryOverheadCheck=false, " +
+                    "intermediatePersistPeriod=PT1H, " +
+                    "maxPendingPersists=100, " +
+                    "indexSpec=IndexSpec{" +
+                    "bitmapSerdeFactory=RoaringBitmapSerdeFactory{}, " +
+                    "dimensionCompression=lz4, " +
+                    "stringDictionaryEncoding=Utf8{}, " +
+                    "metricCompression=lz4, " +
+                    "longEncoding=longs, " +
+                    "complexMetricCompression=null, " +
+                    "jsonCompression=null, " +
+                    "segmentLoader=null" +
+                    "}, " +
+                    "reportParseExceptions=true, " +
+                    "handoffConditionTimeout=100, " +
+                    "resetOffsetAutomatically=false, " +
+                    "segmentWriteOutMediumFactory=null, " +
+                    "workerThreads=null, " +
+                    "chatRetries=8, " +
+                    "httpTimeout=PT10S, " +
+                    "shutdownTimeout=PT80S, " +
+                    "recordBufferSize=1000, " +
+                    "recordBufferOfferTimeout=500, " +
+                    "offsetFetchPeriod=PT30S, " +
+                    "intermediateHandoffPeriod=" + config.getIntermediateHandoffPeriod() + ", " +
+                    "logParseExceptions=false, " +
+                    "maxParseExceptions=0, " +
+                    "maxSavedParseExceptions=0, " +
+                    "numPersistThreads=1, " +
+                    "maxRecordsPerPoll=null}";
+
 
     Assert.assertEquals(resStr, config.toString());
   }
diff --git a/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramComplexMetricSerde.java b/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramComplexMetricSerde.java
index ffad30dd81c..fe937680959 100644
--- a/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramComplexMetricSerde.java
+++ b/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramComplexMetricSerde.java
@@ -21,6 +21,7 @@ package org.apache.druid.spectator.histogram;
 
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.data.ObjectStrategy;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
@@ -97,7 +98,11 @@ public class SpectatorHistogramComplexMetricSerde extends ComplexMetricSerde
   }
 
   @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
+  public GenericColumnSerializer getSerializer(
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      String column,
+      IndexSpec indexSpec
+  )
   {
     return SpectatorHistogramSerializer.create(
         segmentWriteOutMedium,
diff --git a/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramIndexBasedComplexColumn.java b/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramIndexBasedComplexColumn.java
index 8965b595e59..1442da23f26 100644
--- a/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramIndexBasedComplexColumn.java
+++ b/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramIndexBasedComplexColumn.java
@@ -21,6 +21,8 @@ package org.apache.druid.spectator.histogram;
 
 import org.apache.druid.segment.column.ComplexColumn;
 
+import javax.annotation.Nullable;
+
 public class SpectatorHistogramIndexBasedComplexColumn implements ComplexColumn
 {
   private final SpectatorHistogramIndexed index;
@@ -45,6 +47,7 @@ public class SpectatorHistogramIndexBasedComplexColumn implements ComplexColumn
   }
 
   @Override
+  @Nullable
   public Object getRowValue(int rowNum)
   {
     return index.get(rowNum);
diff --git a/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramObjectStrategy.java b/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramObjectStrategy.java
index 33b59bd6ad6..4ffe0a50adc 100644
--- a/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramObjectStrategy.java
+++ b/extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramObjectStrategy.java
@@ -57,4 +57,10 @@ public class SpectatorHistogramObjectStrategy implements ObjectStrategy<Spectato
   {
     return SpectatorHistogramAggregatorFactory.COMPARATOR.compare(o1, o2);
   }
+
+  @Override
+  public boolean readRetainsBufferReference()
+  {
+    return false;
+  }
 }
diff --git a/extensions-contrib/spectator-histogram/src/test/java/org/apache/druid/spectator/histogram/SpectatorHistogramTest.java b/extensions-contrib/spectator-histogram/src/test/java/org/apache/druid/spectator/histogram/SpectatorHistogramTest.java
index fb15ac85e4c..77b81f39847 100644
--- a/extensions-contrib/spectator-histogram/src/test/java/org/apache/druid/spectator/histogram/SpectatorHistogramTest.java
+++ b/extensions-contrib/spectator-histogram/src/test/java/org/apache/druid/spectator/histogram/SpectatorHistogramTest.java
@@ -267,6 +267,7 @@ public class SpectatorHistogramTest
   {
     SegmentWriteOutMedium medium = new OnHeapMemorySegmentWriteOutMedium();
     SpectatorHistogramObjectStrategy strategy = new SpectatorHistogramObjectStrategy();
+    Assert.assertFalse(strategy.readRetainsBufferReference());
     SpectatorHistogramSerializer serializer = SpectatorHistogramSerializer.create(medium, "test", strategy);
     serializer.open();
 
diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchComplexMetricSerde.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchComplexMetricSerde.java
index d44ce80fc7e..b250a66f2d8 100644
--- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchComplexMetricSerde.java
+++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchComplexMetricSerde.java
@@ -22,17 +22,9 @@ package org.apache.druid.query.aggregation.tdigestsketch;
 import com.tdunning.math.stats.MergingDigest;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
-
-import java.nio.ByteBuffer;
 
 public class TDigestSketchComplexMetricSerde extends ComplexMetricSerde
 {
@@ -81,31 +73,9 @@ public class TDigestSketchComplexMetricSerde extends ComplexMetricSerde
     };
   }
 
-  @Override
-  public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder)
-  {
-    final GenericIndexed<MergingDigest> column = GenericIndexed.read(
-        buffer,
-        STRATEGY,
-        builder.getFileMapper()
-    );
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
   @Override
   public ObjectStrategy<MergingDigest> getObjectStrategy()
   {
     return STRATEGY;
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(
-        segmentWriteOutMedium,
-        column,
-        this.getObjectStrategy()
-    );
-  }
-
 }
diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchObjectStrategy.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchObjectStrategy.java
index 706e3e1ac86..1a660d90aff 100644
--- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchObjectStrategy.java
+++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchObjectStrategy.java
@@ -60,4 +60,10 @@ public class TDigestSketchObjectStrategy implements ObjectStrategy<MergingDigest
   {
     return TDigestSketchAggregatorFactory.COMPARATOR.compare(o1, o2);
   }
+
+  @Override
+  public boolean readRetainsBufferReference()
+  {
+    return false;
+  }
 }
diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchObjectStrategyTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchObjectStrategyTest.java
new file mode 100644
index 00000000000..e91809506e9
--- /dev/null
+++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchObjectStrategyTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.aggregation.tdigestsketch;
+
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+public class TDigestSketchObjectStrategyTest
+{
+  @Test
+  public void testReadRetainsBufferReference()
+  {
+    TDigestSketchObjectStrategy strategy = new TDigestSketchObjectStrategy();
+    Assert.assertFalse(strategy.readRetainsBufferReference());
+  }
+}
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeComplexMetricSerde.java
index e9c64feaa64..574c15f3cab 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeComplexMetricSerde.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeComplexMetricSerde.java
@@ -21,18 +21,10 @@ package org.apache.druid.query.aggregation.datasketches.hll;
 
 import org.apache.datasketches.hll.HllSketch;
 import org.apache.druid.data.input.InputRow;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
 import org.apache.druid.segment.data.SafeWritableMemory;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
-
-import java.nio.ByteBuffer;
 
 public class HllSketchMergeComplexMetricSerde extends ComplexMetricSerde
 {
@@ -74,22 +66,4 @@ public class HllSketchMergeComplexMetricSerde extends ComplexMetricSerde
       }
     };
   }
-
-  @Override
-  public void deserializeColumn(final ByteBuffer buf, final ColumnBuilder columnBuilder)
-  {
-    columnBuilder.setComplexColumnSupplier(
-        new ComplexColumnPartSupplier(
-            getTypeName(),
-            GenericIndexed.read(buf, HllSketchHolderObjectStrategy.STRATEGY, columnBuilder.getFileMapper())
-        )
-    );
-  }
-
-  @Override
-  public GenericColumnSerializer getSerializer(final SegmentWriteOutMedium segmentWriteOutMedium, final String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
-
 }
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/kll/KllDoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/kll/KllDoublesSketchComplexMetricSerde.java
index e5249853ac3..1951a421440 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/kll/KllDoublesSketchComplexMetricSerde.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/kll/KllDoublesSketchComplexMetricSerde.java
@@ -23,17 +23,9 @@ import com.google.common.primitives.Doubles;
 import org.apache.datasketches.kll.KllDoublesSketch;
 import org.apache.datasketches.memory.Memory;
 import org.apache.druid.data.input.InputRow;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
-
-import java.nio.ByteBuffer;
 
 public class KllDoublesSketchComplexMetricSerde extends ComplexMetricSerde
 {
@@ -95,18 +87,4 @@ public class KllDoublesSketchComplexMetricSerde extends ComplexMetricSerde
       }
     };
   }
-
-  @Override
-  public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder)
-  {
-    final GenericIndexed<KllDoublesSketch> column = GenericIndexed.read(buffer, STRATEGY, builder.getFileMapper());
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
-  // support large columns
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
 }
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/kll/KllFloatsSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/kll/KllFloatsSketchComplexMetricSerde.java
index 175b307ec33..8e95b04db23 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/kll/KllFloatsSketchComplexMetricSerde.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/kll/KllFloatsSketchComplexMetricSerde.java
@@ -23,17 +23,9 @@ import com.google.common.primitives.Floats;
 import org.apache.datasketches.kll.KllFloatsSketch;
 import org.apache.datasketches.memory.Memory;
 import org.apache.druid.data.input.InputRow;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
-
-import java.nio.ByteBuffer;
 
 public class KllFloatsSketchComplexMetricSerde extends ComplexMetricSerde
 {
@@ -95,18 +87,4 @@ public class KllFloatsSketchComplexMetricSerde extends ComplexMetricSerde
       }
     };
   }
-
-  @Override
-  public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder)
-  {
-    final GenericIndexed<KllFloatsSketch> column = GenericIndexed.read(buffer, STRATEGY, builder.getFileMapper());
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
-  // support large columns
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
 }
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java
index 3614f214c70..8e68a9d69ad 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java
@@ -24,17 +24,9 @@ import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.quantiles.DoublesSketch;
 import org.apache.datasketches.quantiles.UpdateDoublesSketch;
 import org.apache.druid.data.input.InputRow;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
-
-import java.nio.ByteBuffer;
 
 public class DoublesSketchComplexMetricSerde extends ComplexMetricSerde
 {
@@ -96,18 +88,4 @@ public class DoublesSketchComplexMetricSerde extends ComplexMetricSerde
       }
     };
   }
-
-  @Override
-  public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder)
-  {
-    final GenericIndexed<DoublesSketch> column = GenericIndexed.read(buffer, STRATEGY, builder.getFileMapper());
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
-  // support large columns
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
 }
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java
index 4f3ecfae291..c63b3ceb954 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java
@@ -20,18 +20,11 @@
 package org.apache.druid.query.aggregation.datasketches.theta;
 
 import org.apache.druid.data.input.InputRow;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import javax.annotation.Nullable;
-import java.nio.ByteBuffer;
 
 public class SketchMergeComplexMetricSerde extends ComplexMetricSerde
 {
@@ -64,23 +57,9 @@ public class SketchMergeComplexMetricSerde extends ComplexMetricSerde
     };
   }
 
-  @Override
-  public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder)
-  {
-    GenericIndexed<SketchHolder> ge = GenericIndexed.read(buffer, strategy, builder.getFileMapper());
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), ge));
-  }
-
   @Override
   public ObjectStrategy<SketchHolder> getObjectStrategy()
   {
     return strategy;
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
-
 }
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java
index 18371d47926..c2b99a0b6ac 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java
@@ -21,17 +21,9 @@ package org.apache.druid.query.aggregation.datasketches.tuple;
 
 import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
 import org.apache.druid.data.input.InputRow;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
-
-import java.nio.ByteBuffer;
 
 public class ArrayOfDoublesSketchMergeComplexMetricSerde extends ComplexMetricSerde
 {
@@ -65,23 +57,9 @@ public class ArrayOfDoublesSketchMergeComplexMetricSerde extends ComplexMetricSe
     };
   }
 
-  @Override
-  public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder)
-  {
-    final GenericIndexed<ArrayOfDoublesSketch> ge = GenericIndexed.read(buffer, ArrayOfDoublesSketchObjectStrategy.STRATEGY, builder.getFileMapper());
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), ge));
-  }
-
   @Override
   public ObjectStrategy<ArrayOfDoublesSketch> getObjectStrategy()
   {
     return ArrayOfDoublesSketchObjectStrategy.STRATEGY;
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(final SegmentWriteOutMedium segmentWriteOutMedium, final String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
-
 }
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchHolderObjectStrategyTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchHolderObjectStrategyTest.java
index c25135ef2d3..4d5dad537e1 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchHolderObjectStrategyTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchHolderObjectStrategyTest.java
@@ -42,6 +42,7 @@ public class HllSketchHolderObjectStrategyTest
 
     ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
     HllSketchHolderObjectStrategy objectStrategy = new HllSketchHolderObjectStrategy();
+    Assert.assertTrue(objectStrategy.readRetainsBufferReference());
 
     // valid sketch should not explode when copied, which reads the memory
     objectStrategy.fromByteBufferSafe(buf, bytes.length).getSketch().copy();
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/kll/KllDoublesSketchComplexMetricSerdeTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/kll/KllDoublesSketchComplexMetricSerdeTest.java
index 730fb54c541..96de22dfd04 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/kll/KllDoublesSketchComplexMetricSerdeTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/kll/KllDoublesSketchComplexMetricSerdeTest.java
@@ -102,6 +102,7 @@ public class KllDoublesSketchComplexMetricSerdeTest
   {
     final KllDoublesSketchComplexMetricSerde serde = new KllDoublesSketchComplexMetricSerde();
     final ObjectStrategy<KllDoublesSketch> objectStrategy = serde.getObjectStrategy();
+    Assert.assertTrue(objectStrategy.readRetainsBufferReference());
 
     KllDoublesSketch sketch = KllDoublesSketch.newHeapInstance();
     sketch.update(1.1);
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/kll/KllFloatsSketchComplexMetricSerdeTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/kll/KllFloatsSketchComplexMetricSerdeTest.java
index ee505fe65b8..e6d994494f7 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/kll/KllFloatsSketchComplexMetricSerdeTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/kll/KllFloatsSketchComplexMetricSerdeTest.java
@@ -102,6 +102,7 @@ public class KllFloatsSketchComplexMetricSerdeTest
   {
     final KllFloatsSketchComplexMetricSerde serde = new KllFloatsSketchComplexMetricSerde();
     final ObjectStrategy<KllFloatsSketch> objectStrategy = serde.getObjectStrategy();
+    Assert.assertTrue(objectStrategy.readRetainsBufferReference());
 
     KllFloatsSketch sketch = KllFloatsSketch.newHeapInstance();
     sketch.update(1.1f);
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerdeTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerdeTest.java
index 980ce8f8357..36a5ec8ea51 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerdeTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerdeTest.java
@@ -108,6 +108,7 @@ public class DoublesSketchComplexMetricSerdeTest
 
     ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
     ObjectStrategy<DoublesSketch> objectStrategy = serde.getObjectStrategy();
+    Assert.assertTrue(objectStrategy.readRetainsBufferReference());
 
     // valid sketch should not explode when copied, which reads the memory
     objectStrategy.fromByteBufferSafe(buf, bytes.length).toByteArray(true);
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchHolderObjectStrategyTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchHolderObjectStrategyTest.java
index 80b2c9b76bb..3513221f4fe 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchHolderObjectStrategyTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchHolderObjectStrategyTest.java
@@ -36,6 +36,7 @@ public class SketchHolderObjectStrategyTest
   public void testSafeRead()
   {
     SketchHolderObjectStrategy objectStrategy = new SketchHolderObjectStrategy();
+    Assert.assertTrue(objectStrategy.readRetainsBufferReference());
     Union union = (Union) SetOperation.builder().setNominalEntries(1024).build(Family.UNION);
     union.update(1234L);
 
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchObjectStrategyTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchObjectStrategyTest.java
index ee59ddf5764..aab4ebf5772 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchObjectStrategyTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchObjectStrategyTest.java
@@ -33,6 +33,7 @@ public class ArrayOfDoublesSketchObjectStrategyTest
   public void testSafeRead()
   {
     ArrayOfDoublesSketchObjectStrategy objectStrategy = new ArrayOfDoublesSketchObjectStrategy();
+    Assert.assertTrue(objectStrategy.readRetainsBufferReference());
     ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setNominalEntries(1024)
                                                                                      .setNumberOfValues(4)
                                                                                      .build();
diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterSerde.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterSerde.java
index 7a488e05cec..8a9b5371a57 100644
--- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterSerde.java
+++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterSerde.java
@@ -22,6 +22,7 @@ package org.apache.druid.query.aggregation.bloom;
 import org.apache.druid.guice.BloomFilterSerializersModule;
 import org.apache.druid.query.filter.BloomKFilter;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.data.ObjectStrategy;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
@@ -58,7 +59,11 @@ public class BloomFilterSerde extends ComplexMetricSerde
   }
 
   @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
+  public GenericColumnSerializer getSerializer(
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      String column,
+      IndexSpec indexSpec
+  )
   {
     throw new UnsupportedOperationException("Bloom filter aggregators are query-time only");
   }
diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java
index f606fbaf29f..dd35874b8e2 100644
--- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java
+++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java
@@ -22,15 +22,9 @@ package org.apache.druid.query.aggregation.histogram;
 import it.unimi.dsi.fastutil.bytes.ByteArrays;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.Rows;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import java.nio.ByteBuffer;
 import java.util.Collection;
@@ -85,20 +79,6 @@ public class ApproximateHistogramFoldingSerde extends ComplexMetricSerde
     };
   }
 
-  @Override
-  public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder)
-  {
-    final GenericIndexed<ApproximateHistogram> column =
-        GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper());
-    columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
-
   @Override
   public ObjectStrategy<ApproximateHistogram> getObjectStrategy()
   {
@@ -131,6 +111,12 @@ public class ApproximateHistogramFoldingSerde extends ComplexMetricSerde
       {
         return ApproximateHistogramAggregator.COMPARATOR.compare(o1, o2);
       }
+
+      @Override
+      public boolean readRetainsBufferReference()
+      {
+        return false;
+      }
     };
   }
 }
diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramSerde.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramSerde.java
index 5a8e35a71ca..06e62954416 100644
--- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramSerde.java
+++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramSerde.java
@@ -25,15 +25,9 @@ import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.Rows;
 import org.apache.druid.java.util.common.parsers.ParseException;
 import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import javax.annotation.Nullable;
 import java.nio.ByteBuffer;
@@ -131,13 +125,6 @@ public class FixedBucketsHistogramSerde extends ComplexMetricSerde
     };
   }
 
-  @Override
-  public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder)
-  {
-    final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), builder.getFileMapper());
-    builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
   @Override
   public ObjectStrategy getObjectStrategy()
   {
@@ -172,15 +159,12 @@ public class FixedBucketsHistogramSerde extends ComplexMetricSerde
       {
         return comparator.compare(o1, o2);
       }
+
+      @Override
+      public boolean readRetainsBufferReference()
+      {
+        return false;
+      }
     };
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(
-      SegmentWriteOutMedium segmentWriteOutMedium,
-      String column
-  )
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
 }
diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerdeTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerdeTest.java
index d25b17485a3..573f2e54164 100644
--- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerdeTest.java
+++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerdeTest.java
@@ -21,6 +21,7 @@ package org.apache.druid.query.aggregation.histogram;
 
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.data.input.MapBasedInputRow;
+import org.apache.druid.segment.data.ObjectStrategy;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.junit.Assert;
 import org.junit.Test;
@@ -75,6 +76,15 @@ public class ApproximateHistogramFoldingSerdeTest
     );
   }
 
+  @Test
+  public void testReadRetainsBufferReference()
+  {
+    final ApproximateHistogramFoldingSerde serde = new ApproximateHistogramFoldingSerde();
+    final ObjectStrategy<ApproximateHistogram> strategy = serde.getObjectStrategy();
+    Assert.assertFalse(strategy.readRetainsBufferReference());
+  }
+
+
   public static ApproximateHistogram makeHistogram(final float... floats)
   {
     final ApproximateHistogram histogram = new ApproximateHistogram();
diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTest.java
index 630d6e86027..9082dbc0cc1 100644
--- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTest.java
+++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.query.aggregation.histogram;
 import org.apache.commons.math3.distribution.NormalDistribution;
 import org.apache.commons.math3.random.JDKRandomGenerator;
 import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.segment.data.ObjectStrategy;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -1503,4 +1504,12 @@ public class FixedBucketsHistogramTest
     FixedBucketsHistogram fromBase64 = FixedBucketsHistogram.fromBase64(asBase64Full);
     Assert.assertEquals(hSparse, fromBase64);
   }
+
+  @Test
+  public void testObjectStrategyReadRetainsBufferReference()
+  {
+    FixedBucketsHistogramSerde serde = new FixedBucketsHistogramSerde();
+    ObjectStrategy<?> strategy = serde.getObjectStrategy();
+    Assert.assertFalse(strategy.readRetainsBufferReference());
+  }
 }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
index 798a27a95e4..ff2b15121da 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java
@@ -2650,7 +2650,7 @@ public class MSQReplaceTest extends MSQTestBase
       );
     }
 
-    IndexSpec indexSpec = new IndexSpec(null, null, null, null, null, null, null);
+    IndexSpec indexSpec = IndexSpec.DEFAULT;
     GranularitySpec granularitySpec = new UniformGranularitySpec(
         segmentGranularity.getDefaultGranularity(),
         GranularityType.NONE.getDefaultGranularity(),
diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceSerde.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceSerde.java
index f5c081ecfad..5b683227800 100644
--- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceSerde.java
+++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceSerde.java
@@ -21,15 +21,9 @@ package org.apache.druid.query.aggregation.variance;
 
 import com.google.common.collect.Ordering;
 import org.apache.druid.data.input.InputRow;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -82,13 +76,6 @@ public class VarianceSerde extends ComplexMetricSerde
     };
   }
 
-  @Override
-  public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder)
-  {
-    final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper());
-    columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
   @Override
   public ObjectStrategy getObjectStrategy()
   {
@@ -118,13 +105,12 @@ public class VarianceSerde extends ComplexMetricSerde
       {
         return COMPARATOR.compare(o1, o2);
       }
+
+      @Override
+      public boolean readRetainsBufferReference()
+      {
+        return false;
+      }
     };
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
-
 }
diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceSerdeTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceSerdeTest.java
index 4527c8661a6..d23c0473086 100644
--- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceSerdeTest.java
+++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceSerdeTest.java
@@ -35,6 +35,7 @@ public class VarianceSerdeTest
     Random r = ThreadLocalRandom.current();
     VarianceAggregatorCollector holder = new VarianceAggregatorCollector();
     ObjectStrategy strategy = new VarianceSerde().getObjectStrategy();
+    Assert.assertFalse(strategy.readRetainsBufferReference());
     Assert.assertEquals(VarianceAggregatorCollector.class, strategy.getClazz());
 
     for (int i = 0; i < 100; i++) {
diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/ComplexFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/ComplexFrameColumnReader.java
index 23f3a7d7432..f22aeb8000b 100644
--- a/processing/src/main/java/org/apache/druid/frame/read/columnar/ComplexFrameColumnReader.java
+++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/ComplexFrameColumnReader.java
@@ -185,6 +185,7 @@ public class ComplexFrameColumnReader implements FrameColumnReader
     }
 
     @Override
+    @Nullable
     public Object getRowValue(int rowNum)
     {
       // Need bounds checking, since getObjectForPhysicalRow doesn't do it.
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexColumn.java
index 56fac598ee4..0c2f513ba11 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexColumn.java
@@ -26,6 +26,7 @@ import org.apache.druid.segment.column.ComplexColumn;
 import org.apache.druid.segment.serde.cell.ByteBufferProvider;
 import org.apache.druid.segment.serde.cell.CellReader;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -63,6 +64,7 @@ public class SerializablePairLongDoubleComplexColumn implements ComplexColumn
   }
 
   @Override
+  @Nullable
   public Object getRowValue(int rowNum)
   {
     return serde.deserialize(cellReader.getCell(rowNum));
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java
index 1d5e81d3ac6..15b36d6182d 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java
@@ -22,6 +22,7 @@ package org.apache.druid.query.aggregation;
 import it.unimi.dsi.fastutil.Hash;
 import org.apache.druid.collections.SerializablePair;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ObjectStrategyComplexTypeStrategy;
@@ -58,7 +59,11 @@ public class SerializablePairLongDoubleComplexMetricSerde extends AbstractSerial
   }
 
   @Override
-  public GenericColumnSerializer<SerializablePairLongDouble> getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
+  public GenericColumnSerializer<SerializablePairLongDouble> getSerializer(
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      String column,
+      IndexSpec indexSpec
+  )
   {
     return new SerializablePairLongDoubleColumnSerializer(
         segmentWriteOutMedium,
@@ -108,6 +113,12 @@ public class SerializablePairLongDoubleComplexMetricSerde extends AbstractSerial
       {
         return SERDE.serialize(inPair);
       }
+
+      @Override
+      public boolean readRetainsBufferReference()
+      {
+        return false;
+      }
     };
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java
index fa841b24072..9b437728b30 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java
@@ -26,6 +26,7 @@ import org.apache.druid.segment.column.ComplexColumn;
 import org.apache.druid.segment.serde.cell.ByteBufferProvider;
 import org.apache.druid.segment.serde.cell.CellReader;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -63,6 +64,7 @@ public class SerializablePairLongFloatComplexColumn implements ComplexColumn
   }
 
   @Override
+  @Nullable
   public Object getRowValue(int rowNum)
   {
     return serde.deserialize(cellReader.getCell(rowNum));
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java
index 1f4f624fef4..4df89153caf 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java
@@ -22,6 +22,7 @@ package org.apache.druid.query.aggregation;
 import it.unimi.dsi.fastutil.Hash;
 import org.apache.druid.collections.SerializablePair;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ObjectStrategyComplexTypeStrategy;
@@ -58,7 +59,11 @@ public class SerializablePairLongFloatComplexMetricSerde extends AbstractSeriali
   }
 
   @Override
-  public GenericColumnSerializer<SerializablePairLongFloat> getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
+  public GenericColumnSerializer<SerializablePairLongFloat> getSerializer(
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      String column,
+      IndexSpec indexSpec
+  )
   {
     return new SerializablePairLongFloatColumnSerializer(
         segmentWriteOutMedium,
@@ -109,6 +114,12 @@ public class SerializablePairLongFloatComplexMetricSerde extends AbstractSeriali
       {
         return SERDE.serialize(inPair);
       }
+
+      @Override
+      public boolean readRetainsBufferReference()
+      {
+        return false;
+      }
     };
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java
index b5a3806f564..5ffdd98f264 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java
@@ -26,6 +26,7 @@ import org.apache.druid.segment.column.ComplexColumn;
 import org.apache.druid.segment.serde.cell.ByteBufferProvider;
 import org.apache.druid.segment.serde.cell.CellReader;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -63,6 +64,7 @@ public class SerializablePairLongLongComplexColumn implements ComplexColumn
   }
 
   @Override
+  @Nullable
   public Object getRowValue(int rowNum)
   {
     return serde.deserialize(cellReader.getCell(rowNum));
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java
index 7b8a6004015..00f4c6a6ab8 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java
@@ -22,6 +22,7 @@ package org.apache.druid.query.aggregation;
 import it.unimi.dsi.fastutil.Hash;
 import org.apache.druid.collections.SerializablePair;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ObjectStrategyComplexTypeStrategy;
@@ -58,7 +59,11 @@ public class SerializablePairLongLongComplexMetricSerde extends AbstractSerializ
   }
 
   @Override
-  public GenericColumnSerializer<SerializablePairLongLong> getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
+  public GenericColumnSerializer<SerializablePairLongLong> getSerializer(
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      String column,
+      IndexSpec indexSpec
+  )
   {
     return new SerializablePairLongLongColumnSerializer(
         segmentWriteOutMedium,
@@ -108,6 +113,12 @@ public class SerializablePairLongLongComplexMetricSerde extends AbstractSerializ
       {
         return SERDE.serialize(inPair);
       }
+
+      @Override
+      public boolean readRetainsBufferReference()
+      {
+        return false;
+      }
     };
   }
 
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexColumn.java
index 264dfc4a853..81ac5717705 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexColumn.java
@@ -26,6 +26,7 @@ import org.apache.druid.segment.column.ComplexColumn;
 import org.apache.druid.segment.serde.cell.ByteBufferProvider;
 import org.apache.druid.segment.serde.cell.CellReader;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -64,6 +65,7 @@ public class SerializablePairLongStringComplexColumn implements ComplexColumn
 
   @SuppressWarnings("ConstantConditions")
   @Override
+  @Nullable
   public Object getRowValue(int rowNum)
   {
     // This can return nulls, meaning that it is expected that anything reading from this does
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerde.java
index 625d597ba8c..fa393a5f220 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerde.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerde.java
@@ -24,6 +24,7 @@ import org.apache.druid.collections.SerializablePair;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ObjectStrategyComplexTypeStrategy;
@@ -193,7 +194,11 @@ public class SerializablePairLongStringComplexMetricSerde extends ComplexMetricS
   }
 
   @Override
-  public GenericColumnSerializer<?> getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
+  public GenericColumnSerializer<?> getSerializer(
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      String column,
+      IndexSpec indexSpec
+  )
   {
     if (compressionEnabled) {
       return new SerializablePairLongStringColumnSerializer(
@@ -263,5 +268,11 @@ public class SerializablePairLongStringComplexMetricSerde extends ComplexMetricS
 
           return bbuf.array();
         }
+
+        @Override
+        public boolean readRetainsBufferReference()
+        {
+          return false;
+        }
       };
 }
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java
index 26e3140f829..eabcc433330 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java
@@ -22,15 +22,9 @@ package org.apache.druid.query.aggregation.hyperloglog;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.hll.HyperLogLogCollector;
 import org.apache.druid.hll.HyperLogLogHash;
-import org.apache.druid.segment.GenericColumnSerializer;
-import org.apache.druid.segment.column.ColumnBuilder;
-import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
-import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import java.nio.ByteBuffer;
 import java.util.Comparator;
@@ -96,13 +90,6 @@ public class HyperUniquesSerde extends ComplexMetricSerde
     };
   }
 
-  @Override
-  public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder)
-  {
-    final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper());
-    columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column));
-  }
-
   @Override
   public ObjectStrategy getObjectStrategy()
   {
@@ -143,11 +130,4 @@ public class HyperUniquesSerde extends ComplexMetricSerde
       }
     };
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
-  }
-
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java
index c60a4acf351..fd3400d6c1f 100644
--- a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java
+++ b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java
@@ -732,7 +732,7 @@ public class IndexMergerV9 implements IndexMerger
           if (serde == null) {
             throw new ISE("Unknown type[%s]", type.getComplexTypeName());
           }
-          writer = serde.getSerializer(segmentWriteOutMedium, metric);
+          writer = serde.getSerializer(segmentWriteOutMedium, metric, indexSpec);
           break;
         default:
           throw new ISE("Unknown type[%s]", type);
diff --git a/processing/src/main/java/org/apache/druid/segment/IndexSpec.java b/processing/src/main/java/org/apache/druid/segment/IndexSpec.java
index 1c8b097d523..37a7e6cc0d6 100644
--- a/processing/src/main/java/org/apache/druid/segment/IndexSpec.java
+++ b/processing/src/main/java/org/apache/druid/segment/IndexSpec.java
@@ -55,7 +55,8 @@ public class IndexSpec
   private final StringEncodingStrategy stringDictionaryEncoding;
   private final CompressionStrategy metricCompression;
   private final CompressionFactory.LongEncodingStrategy longEncoding;
-
+  @Nullable
+  private final CompressionStrategy complexMetricCompression;
   @Nullable
   private final CompressionStrategy jsonCompression;
   @Nullable
@@ -84,6 +85,7 @@ public class IndexSpec
       @JsonProperty("stringDictionaryEncoding") @Nullable StringEncodingStrategy stringDictionaryEncoding,
       @JsonProperty("metricCompression") @Nullable CompressionStrategy metricCompression,
       @JsonProperty("longEncoding") @Nullable CompressionFactory.LongEncodingStrategy longEncoding,
+      @JsonProperty("complexMetricCompression") @Nullable CompressionStrategy complexMetricCompression,
       @JsonProperty("jsonCompression") @Nullable CompressionStrategy jsonCompression,
       @JsonProperty("segmentLoader") @Nullable SegmentizerFactory segmentLoader
   )
@@ -101,6 +103,7 @@ public class IndexSpec
     this.metricCompression = metricCompression == null
                              ? CompressionStrategy.DEFAULT_COMPRESSION_STRATEGY
                              : metricCompression;
+    this.complexMetricCompression = complexMetricCompression;
     this.longEncoding = longEncoding == null
                         ? CompressionFactory.DEFAULT_LONG_ENCODING_STRATEGY
                         : longEncoding;
@@ -138,6 +141,14 @@ public class IndexSpec
     return longEncoding;
   }
 
+  @JsonProperty
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  @Nullable
+  public CompressionStrategy getComplexMetricCompression()
+  {
+    return complexMetricCompression;
+  }
+
   @JsonProperty
   @JsonInclude(JsonInclude.Include.NON_NULL)
   @Nullable
@@ -179,6 +190,7 @@ public class IndexSpec
            Objects.equals(stringDictionaryEncoding, indexSpec.stringDictionaryEncoding) &&
            metricCompression == indexSpec.metricCompression &&
            longEncoding == indexSpec.longEncoding &&
+           Objects.equals(complexMetricCompression, indexSpec.complexMetricCompression) &&
            Objects.equals(jsonCompression, indexSpec.jsonCompression) &&
            Objects.equals(segmentLoader, indexSpec.segmentLoader);
   }
@@ -192,6 +204,7 @@ public class IndexSpec
         stringDictionaryEncoding,
         metricCompression,
         longEncoding,
+        complexMetricCompression,
         jsonCompression,
         segmentLoader
     );
@@ -206,6 +219,7 @@ public class IndexSpec
            ", stringDictionaryEncoding=" + stringDictionaryEncoding +
            ", metricCompression=" + metricCompression +
            ", longEncoding=" + longEncoding +
+           ", complexMetricCompression=" + complexMetricCompression +
            ", jsonCompression=" + jsonCompression +
            ", segmentLoader=" + segmentLoader +
            '}';
@@ -224,6 +238,8 @@ public class IndexSpec
     @Nullable
     private CompressionFactory.LongEncodingStrategy longEncoding;
     @Nullable
+    private CompressionStrategy complexMetricCompression;
+    @Nullable
     private CompressionStrategy jsonCompression;
     @Nullable
     private SegmentizerFactory segmentLoader;
@@ -252,6 +268,12 @@ public class IndexSpec
       return this;
     }
 
+    public Builder withComplexMetricCompression(CompressionStrategy complexMetricCompression)
+    {
+      this.complexMetricCompression = complexMetricCompression;
+      return this;
+    }
+
     public Builder withLongEncoding(CompressionFactory.LongEncodingStrategy longEncoding)
     {
       this.longEncoding = longEncoding;
@@ -278,6 +300,7 @@ public class IndexSpec
           stringDictionaryEncoding,
           metricCompression,
           longEncoding,
+          complexMetricCompression,
           jsonCompression,
           segmentLoader
       );
diff --git a/processing/src/main/java/org/apache/druid/segment/column/ComplexColumn.java b/processing/src/main/java/org/apache/druid/segment/column/ComplexColumn.java
index b0f3a4388c8..6204d7d219d 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/ComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/ComplexColumn.java
@@ -52,9 +52,11 @@ public interface ComplexColumn extends BaseColumn
 
   /**
    * Return rows in the column.
+   *
    * @param rowNum the row number
    * @return row object of type same as {@link ComplexColumn#getClazz()}  } at row number "rowNum" .
    */
+  @Nullable
   Object getRowValue(int rowNum);
 
   /**
diff --git a/processing/src/main/java/org/apache/druid/segment/column/GenericIndexedBasedComplexColumn.java b/processing/src/main/java/org/apache/druid/segment/column/GenericIndexedBasedComplexColumn.java
index 4b1654d8884..c2649348256 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/GenericIndexedBasedComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/GenericIndexedBasedComplexColumn.java
@@ -21,6 +21,8 @@ package org.apache.druid.segment.column;
 
 import org.apache.druid.segment.data.GenericIndexed;
 
+import javax.annotation.Nullable;
+
 /**
  * Implementation of {@link ComplexColumn} to be used when complex column serialization is done by using
  * {@link GenericIndexed} by using default implementations of "writeToXXX" methods in
@@ -50,6 +52,7 @@ public class GenericIndexedBasedComplexColumn implements ComplexColumn
   }
 
   @Override
+  @Nullable
   public Object getRowValue(int rowNum)
   {
     return index.get(rowNum);
diff --git a/processing/src/main/java/org/apache/druid/segment/column/NullableTypeStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/NullableTypeStrategy.java
index 88642a964e8..b6d5114fb0a 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/NullableTypeStrategy.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/NullableTypeStrategy.java
@@ -106,9 +106,13 @@ public final class NullableTypeStrategy<T> implements Comparator<T>, Hash.Strate
   }
 
   /**
-   * Whether the {@link #read} methods return an object that may retain a reference to the provided {@link ByteBuffer}.
-   * If a reference is sometimes retained, this method returns true. It returns false if, and only if, a reference
-   * is *never* retained.
+   * Whether the {@link #read} methods return an object that may retain a reference to the underlying memory of the
+   * provided {@link ByteBuffer}. If a reference is sometimes retained, this method returns true. It returns false if,
+   * and only if, a reference is *never* retained.
+   * <p>
+   * If this method returns true, and the caller does not control the lifecycle of the underlying memory or cannot
+   * ensure that it will not change over the lifetime of the returned object, callers should copy the memory to a new
+   * location that they do control the lifecycle of and will be available for the duration of the returned object.
    */
   public boolean readRetainsBufferReference()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java
index f80a1cdcf8d..9a67220580b 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java
@@ -82,8 +82,7 @@ public class ObjectStrategyComplexTypeStrategy<T> implements TypeStrategy<T>
   @Override
   public boolean readRetainsBufferReference()
   {
-    // Can't guarantee that ObjectStrategy *doesn't* retain a reference.
-    return true;
+    return objectStrategy.readRetainsBufferReference();
   }
 
   @Override
diff --git a/processing/src/main/java/org/apache/druid/segment/column/TypeStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/TypeStrategy.java
index 075fceca473..1d75d9fde74 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/TypeStrategy.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/TypeStrategy.java
@@ -87,9 +87,13 @@ public interface TypeStrategy<T> extends Comparator<Object>, Hash.Strategy<T>
   T read(ByteBuffer buffer);
 
   /**
-   * Whether the {@link #read} methods return an object that may retain a reference to the provided {@link ByteBuffer}.
-   * If a reference is sometimes retained, this method returns true. It returns false if, and only if, a reference
-   * is *never* retained.
+   * Whether the {@link #read} methods return an object that may retain a reference to the underlying memory of the
+   * provided {@link ByteBuffer}. If a reference is sometimes retained, this method returns true. It returns false if,
+   * and only if, a reference is *never* retained.
+   * <p>
+   * If this method returns true, and the caller does not control the lifecycle of the underlying memory or cannot
+   * ensure that it will not change over the lifetime of the returned object, callers should copy the memory to a new
+   * location that they do control the lifecycle of and will be available for the duration of the returned object.
    */
   boolean readRetainsBufferReference();
 
diff --git a/processing/src/main/java/org/apache/druid/segment/column/UnknownTypeComplexColumn.java b/processing/src/main/java/org/apache/druid/segment/column/UnknownTypeComplexColumn.java
index 3f11b779eaf..f1b7d0f1af2 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/UnknownTypeComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/UnknownTypeComplexColumn.java
@@ -49,8 +49,8 @@ public class UnknownTypeComplexColumn implements ComplexColumn
     return "UNKNOWN_COMPLEX_COLUMN_TYPE";
   }
 
-  @Nullable
   @Override
+  @Nullable
   public Object getRowValue(int rowNum)
   {
     return null;
diff --git a/processing/src/main/java/org/apache/druid/segment/data/CompressedBlockReader.java b/processing/src/main/java/org/apache/druid/segment/data/CompressedBlockReader.java
index f03eb80fb43..4ab7f1ddb45 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/CompressedBlockReader.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/CompressedBlockReader.java
@@ -56,7 +56,11 @@ public final class CompressedBlockReader implements Closeable
   private static final ByteBuffer NULL_VALUE = ByteBuffer.wrap(new byte[0]);
   public static final byte VERSION = 0x01;
 
-  public static Supplier<CompressedBlockReader> fromByteBuffer(ByteBuffer buffer, ByteOrder byteOrder)
+  public static Supplier<CompressedBlockReader> fromByteBuffer(
+      ByteBuffer buffer,
+      ByteOrder byteOrder,
+      boolean copyValuesOnRead
+  )
   {
     byte versionFromBuffer = buffer.get();
 
@@ -87,6 +91,7 @@ public final class CompressedBlockReader implements Closeable
           compression,
           numBlocks,
           blockSize,
+          copyValuesOnRead,
           offsetView.asReadOnlyBuffer(),
           compressedDataView.asReadOnlyBuffer().order(byteOrder),
           byteOrder
@@ -97,6 +102,7 @@ public final class CompressedBlockReader implements Closeable
 
   private final CompressionStrategy.Decompressor decompressor;
 
+  private final boolean copyValuesOnRead;
   private final int numBlocks;
   private final int div;
   private final int rem;
@@ -114,12 +120,14 @@ public final class CompressedBlockReader implements Closeable
       CompressionStrategy compressionStrategy,
       int numBlocks,
       int blockSize,
+      boolean copyValuesOnRead,
       IntBuffer endOffsetsBuffer,
       ByteBuffer compressedDataBuffer,
       ByteOrder byteOrder
   )
   {
     this.decompressor = compressionStrategy.getDecompressor();
+    this.copyValuesOnRead = copyValuesOnRead;
     this.numBlocks = numBlocks;
     this.div = Integer.numberOfTrailingZeros(blockSize);
     this.rem = blockSize - 1;
@@ -169,18 +177,25 @@ public final class CompressedBlockReader implements Closeable
     if (size == 0) {
       return NULL_VALUE;
     }
-
     final int startBlockOffset = loadBlock(startOffset);
     final int startBlockNumber = currentBlockNumber;
     decompressedDataBuffer.position(startBlockOffset);
-    // patch together value from n underlying compressed pages
+    // possibly patch together value from n underlying compressed pages
     if (size < decompressedDataBuffer.remaining()) {
-      // sweet, same buffer, we can slice out a view directly to the value
-      final ByteBuffer dupe = decompressedDataBuffer.duplicate().order(byteOrder);
-      dupe.position(startBlockOffset).limit(startBlockOffset + size);
-      return dupe.slice().order(byteOrder);
+      // sweet, same buffer
+      if (copyValuesOnRead) {
+        // caller specified copyValuesOnRead, so copy the memory to a heap byte array
+        final byte[] bytes = new byte[size];
+        decompressedDataBuffer.get(bytes, 0, size);
+        return ByteBuffer.wrap(bytes).order(byteOrder);
+      } else {
+        // if we don't need to copy, we can return the buffer directly with position and limit set
+        final ByteBuffer dupe = decompressedDataBuffer.duplicate().order(byteOrder);
+        dupe.position(startBlockOffset).limit(startBlockOffset + size);
+        return dupe;
+      }
     } else {
-      // spans multiple blocks, copy on heap
+      // spans multiple blocks, always copy on heap
       final byte[] bytes = new byte[size];
       int bytesRead = 0;
       int block = startBlockNumber;
diff --git a/processing/src/main/java/org/apache/druid/segment/data/CompressedLongsReader.java b/processing/src/main/java/org/apache/druid/segment/data/CompressedLongsReader.java
index 84b96ea0709..12d97dbfcc3 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/CompressedLongsReader.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/CompressedLongsReader.java
@@ -29,7 +29,7 @@ public final class CompressedLongsReader implements ColumnarLongs
 {
   public static Supplier<CompressedLongsReader> fromByteBuffer(ByteBuffer buffer, ByteOrder order)
   {
-    final Supplier<CompressedBlockReader> baseReader = CompressedBlockReader.fromByteBuffer(buffer, order);
+    final Supplier<CompressedBlockReader> baseReader = CompressedBlockReader.fromByteBuffer(buffer, order, false);
     return () -> new CompressedLongsReader(baseReader.get());
   }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/data/CompressedVariableSizedBlobColumnSupplier.java b/processing/src/main/java/org/apache/druid/segment/data/CompressedVariableSizedBlobColumnSupplier.java
index 84c783e67ea..bf88dbfc8c4 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/CompressedVariableSizedBlobColumnSupplier.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/CompressedVariableSizedBlobColumnSupplier.java
@@ -37,6 +37,17 @@ public class CompressedVariableSizedBlobColumnSupplier implements Supplier<Compr
       ByteOrder order,
       SmooshedFileMapper mapper
   ) throws IOException
+  {
+    return fromByteBuffer(filenameBase, buffer, order, false, mapper);
+  }
+
+  public static CompressedVariableSizedBlobColumnSupplier fromByteBuffer(
+      String filenameBase,
+      ByteBuffer buffer,
+      ByteOrder order,
+      boolean copyValuesOnRead,
+      SmooshedFileMapper mapper
+  ) throws IOException
   {
     byte versionFromBuffer = buffer.get();
     if (versionFromBuffer == VERSION) {
@@ -48,7 +59,7 @@ public class CompressedVariableSizedBlobColumnSupplier implements Supplier<Compr
       final ByteBuffer dataBuffer = mapper.mapFile(
           CompressedVariableSizedBlobColumnSerializer.getCompressedBlobsFileName(filenameBase)
       );
-      return new CompressedVariableSizedBlobColumnSupplier(offsetsBuffer, dataBuffer, order, numElements);
+      return new CompressedVariableSizedBlobColumnSupplier(offsetsBuffer, dataBuffer, order, numElements, copyValuesOnRead);
     }
     throw new IAE("Unknown version[%s]", versionFromBuffer);
   }
@@ -58,16 +69,17 @@ public class CompressedVariableSizedBlobColumnSupplier implements Supplier<Compr
   private final Supplier<CompressedLongsReader> offsetReaderSupplier;
   private final Supplier<CompressedBlockReader> blockDataReaderSupplier;
 
-  public CompressedVariableSizedBlobColumnSupplier(
+  private CompressedVariableSizedBlobColumnSupplier(
       ByteBuffer offsetsBuffer,
       ByteBuffer dataBuffer,
       ByteOrder order,
-      int numElements
+      int numElements,
+      boolean copyValuesOnRead
   )
   {
     this.numElements = numElements;
     this.offsetReaderSupplier = CompressedLongsReader.fromByteBuffer(offsetsBuffer, order);
-    this.blockDataReaderSupplier = CompressedBlockReader.fromByteBuffer(dataBuffer, order);
+    this.blockDataReaderSupplier = CompressedBlockReader.fromByteBuffer(dataBuffer, order, copyValuesOnRead);
   }
 
   @Override
diff --git a/processing/src/main/java/org/apache/druid/segment/data/GenericIndexed.java b/processing/src/main/java/org/apache/druid/segment/data/GenericIndexed.java
index a87dcda09b5..19c58e411f3 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/GenericIndexed.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/GenericIndexed.java
@@ -167,6 +167,12 @@ public abstract class GenericIndexed<T> implements CloseableIndexed<T>, Serializ
     {
       return Comparators.<String>naturalNullsFirst().compare(o1, o2);
     }
+
+    @Override
+    public boolean readRetainsBufferReference()
+    {
+      return false;
+    }
   };
 
   public static <T> GenericIndexed<T> read(ByteBuffer buffer, ObjectStrategy<T> strategy)
diff --git a/processing/src/main/java/org/apache/druid/segment/data/ObjectStrategy.java b/processing/src/main/java/org/apache/druid/segment/data/ObjectStrategy.java
index eba97d04bbb..da95ee099ea 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/ObjectStrategy.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/ObjectStrategy.java
@@ -20,6 +20,7 @@
 package org.apache.druid.segment.data;
 
 import org.apache.druid.guice.annotations.ExtensionPoint;
+import org.apache.druid.segment.column.TypeStrategy;
 import org.apache.druid.segment.writeout.WriteOutBytes;
 
 import javax.annotation.Nullable;
@@ -59,6 +60,23 @@ public interface ObjectStrategy<T> extends Comparator<T>
     return true;
   }
 
+  /**
+   * Whether the {@link #fromByteBuffer(ByteBuffer, int)}, {@link #fromByteBufferWithSize(ByteBuffer)}, and
+   * {@link #fromByteBufferSafe(ByteBuffer, int)} methods return an object that may retain a reference to the underlying
+   * memory provided by a {@link ByteBuffer}. If a reference is sometimes retained, this method returns true. It
+   * returns false if, and only if, a reference is *never* retained.
+   * <p>
+   * If this method returns true, and the caller does not control the lifecycle of the underlying memory or cannot
+   * ensure that it will not change over the lifetime of the returned object, callers should copy the memory to a new
+   * location that they do control the lifecycle of and will be available for the duration of the returned object.
+   *
+   * @see TypeStrategy#readRetainsBufferReference()
+   */
+  default boolean readRetainsBufferReference()
+  {
+    return true;
+  }
+
   /**
    * Reads 4-bytes numBytes from the given buffer, and then delegates to {@link #fromByteBuffer(ByteBuffer, int)}.
    */
diff --git a/processing/src/main/java/org/apache/druid/segment/generator/GeneratorBasicSchemas.java b/processing/src/main/java/org/apache/druid/segment/generator/GeneratorBasicSchemas.java
index 0727309654b..ef2bb096cb9 100644
--- a/processing/src/main/java/org/apache/druid/segment/generator/GeneratorBasicSchemas.java
+++ b/processing/src/main/java/org/apache/druid/segment/generator/GeneratorBasicSchemas.java
@@ -43,6 +43,20 @@ public class GeneratorBasicSchemas
 {
   private static final ImmutableMap.Builder<String, GeneratorSchemaInfo> SCHEMA_INFO_BUILDER = ImmutableMap.builder();
 
+  public static final String BASIC_SCHEMA = "basic";
+  public static final String BASIC_SCHEMA_EXPRESSION = "expression";
+  public static final String SIMPLE_SCHEMA = "simple";
+  public static final String SIMPLE_LONG_SCHEMA = "simpleLong";
+  public static final String SIMPLE_FLOAT_SCHEMA = "simpleFloat";
+  public static final String SIMPLE_NULLS_SCHEMA = "nulls";
+  public static final String ROLLUP_SCHEMA = "rollo";
+  public static final String NULLABLE_TEST_SCHEMA = "nullable";
+  public static final String EXPRESSION_TESTBENCH_SCHEMA = "expression-testbench";
+  public static final String NESTED_TESTBENCH_SCHEMA = "nested";
+  public static final String GROUPBY_TESTBENCH_SCHEMA = "groupBy-testbench";
+  public static final String IN_TESTBENCH_SCHEMA = "in-testbench";
+  public static final String WIDE_SCHEMA = "wide";
+
   static {
     // basic schema
     List<GeneratorColumnSchema> basicSchemaColumns = ImmutableList.of(
@@ -122,8 +136,8 @@ public class GeneratorBasicSchemas
         true
     );
 
-    SCHEMA_INFO_BUILDER.put("basic", basicSchema);
-    SCHEMA_INFO_BUILDER.put("expression", basicSchemaExpression);
+    SCHEMA_INFO_BUILDER.put(BASIC_SCHEMA, basicSchema);
+    SCHEMA_INFO_BUILDER.put(BASIC_SCHEMA_EXPRESSION, basicSchemaExpression);
   }
 
   static { // simple single string column and count agg schema, no rollup
@@ -143,7 +157,7 @@ public class GeneratorBasicSchemas
         basicSchemaDataInterval,
         false
     );
-    SCHEMA_INFO_BUILDER.put("simple", basicSchema);
+    SCHEMA_INFO_BUILDER.put(SIMPLE_SCHEMA, basicSchema);
   }
 
   static {
@@ -165,7 +179,7 @@ public class GeneratorBasicSchemas
         basicSchemaDataInterval,
         false
     );
-    SCHEMA_INFO_BUILDER.put("simpleLong", basicSchema);
+    SCHEMA_INFO_BUILDER.put(SIMPLE_LONG_SCHEMA, basicSchema);
   }
 
   static {
@@ -187,7 +201,7 @@ public class GeneratorBasicSchemas
         basicSchemaDataInterval,
         false
     );
-    SCHEMA_INFO_BUILDER.put("simpleFloat", basicSchema);
+    SCHEMA_INFO_BUILDER.put(SIMPLE_FLOAT_SCHEMA, basicSchema);
   }
 
   static {
@@ -238,7 +252,7 @@ public class GeneratorBasicSchemas
         basicSchemaDataInterval,
         true
     );
-    SCHEMA_INFO_BUILDER.put("rollo", rolloSchema);
+    SCHEMA_INFO_BUILDER.put(ROLLUP_SCHEMA, rolloSchema);
   }
 
   static {
@@ -268,7 +282,7 @@ public class GeneratorBasicSchemas
         false
     );
 
-    SCHEMA_INFO_BUILDER.put("nulls", nullsSchema);
+    SCHEMA_INFO_BUILDER.put(SIMPLE_NULLS_SCHEMA, nullsSchema);
   }
 
   static {
@@ -309,7 +323,7 @@ public class GeneratorBasicSchemas
         false
     );
 
-    SCHEMA_INFO_BUILDER.put("nulls-and-non-nulls", nullsSchema);
+    SCHEMA_INFO_BUILDER.put(NULLABLE_TEST_SCHEMA, nullsSchema);
   }
 
   static {
@@ -362,7 +376,8 @@ public class GeneratorBasicSchemas
         false
     );
 
-    SCHEMA_INFO_BUILDER.put("expression-testbench", expressionsTestsSchema);
+    SCHEMA_INFO_BUILDER.put(EXPRESSION_TESTBENCH_SCHEMA, expressionsTestsSchema);
+    SCHEMA_INFO_BUILDER.put(NESTED_TESTBENCH_SCHEMA, expressionsTestsSchema);
   }
 
   static {
@@ -406,7 +421,7 @@ public class GeneratorBasicSchemas
         false
     );
 
-    SCHEMA_INFO_BUILDER.put("groupBy-testbench", groupByTestsSchema);
+    SCHEMA_INFO_BUILDER.put(GROUPBY_TESTBENCH_SCHEMA, groupByTestsSchema);
   }
 
   static {
@@ -426,7 +441,7 @@ public class GeneratorBasicSchemas
         false
     );
 
-    SCHEMA_INFO_BUILDER.put("in-testbench", expressionsTestsSchema);
+    SCHEMA_INFO_BUILDER.put(IN_TESTBENCH_SCHEMA, expressionsTestsSchema);
 
   }
 
@@ -532,7 +547,7 @@ public class GeneratorBasicSchemas
         false
     );
 
-    SCHEMA_INFO_BUILDER.put("wide", nullsSchema);
+    SCHEMA_INFO_BUILDER.put(WIDE_SCHEMA, nullsSchema);
   }
 
   public static final Map<String, GeneratorSchemaInfo> SCHEMA_MAP = SCHEMA_INFO_BUILDER.build();
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java b/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
index 5f1d765bb56..9821155e178 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
@@ -310,8 +310,8 @@ public abstract class CompressedNestedDataComplexColumn<TStringDictionary extend
     return nullValues;
   }
 
-  @Nullable
   @Override
+  @Nullable
   public Object getRowValue(int rowNum)
   {
     if (nullValues.get(rowNum)) {
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java b/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java
index d5e8a4ab6f4..bfccf74c17b 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java
@@ -34,6 +34,7 @@ import org.apache.druid.segment.data.DictionaryWriter;
 import org.apache.druid.segment.data.FixedIndexed;
 import org.apache.druid.segment.data.FrontCodedIntArrayIndexed;
 import org.apache.druid.segment.data.Indexed;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 
 import javax.annotation.Nullable;
 import java.io.Closeable;
@@ -227,9 +228,9 @@ public final class DictionaryIdLookup implements Closeable
       // mapper so that we can have a mutable smoosh)
       File stringSmoosh = FileUtils.createTempDirInLocation(tempBasePath, StringUtils.urlEncode(name) + "__stringTempSmoosh");
       stringDictionaryFile = stringSmoosh.toPath();
-      final String fileName = NestedCommonFormatColumnSerializer.getInternalFileName(
+      final String fileName = ColumnSerializerUtils.getInternalFileName(
           name,
-          NestedCommonFormatColumnSerializer.STRING_DICTIONARY_FILE_NAME
+          ColumnSerializerUtils.STRING_DICTIONARY_FILE_NAME
       );
 
       try (
@@ -259,7 +260,7 @@ public final class DictionaryIdLookup implements Closeable
   private void ensureLongDictionaryLoaded()
   {
     if (longDictionary == null) {
-      longDictionaryFile = makeTempFile(name + NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME);
+      longDictionaryFile = makeTempFile(name + ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME);
       longBuffer = mapWriter(longDictionaryFile, longDictionaryWriter);
       longDictionary = FixedIndexed.read(longBuffer, TypeStrategies.LONG, ByteOrder.nativeOrder(), Long.BYTES).get();
       // reset position
@@ -270,7 +271,7 @@ public final class DictionaryIdLookup implements Closeable
   private void ensureDoubleDictionaryLoaded()
   {
     if (doubleDictionary == null) {
-      doubleDictionaryFile = makeTempFile(name + NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME);
+      doubleDictionaryFile = makeTempFile(name + ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME);
       doubleBuffer = mapWriter(doubleDictionaryFile, doubleDictionaryWriter);
       doubleDictionary = FixedIndexed.read(
           doubleBuffer,
@@ -286,7 +287,7 @@ public final class DictionaryIdLookup implements Closeable
   private void ensureArrayDictionaryLoaded()
   {
     if (arrayDictionary == null && arrayDictionaryWriter != null) {
-      arrayDictionaryFile = makeTempFile(name + NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME);
+      arrayDictionaryFile = makeTempFile(name + ColumnSerializerUtils.ARRAY_DICTIONARY_FILE_NAME);
       arrayBuffer = mapWriter(arrayDictionaryFile, arrayDictionaryWriter);
       arrayDictionary = FrontCodedIntArrayIndexed.read(arrayBuffer, ByteOrder.nativeOrder()).get();
       // reset position
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/GlobalDictionaryEncodedFieldColumnWriter.java b/processing/src/main/java/org/apache/druid/segment/nested/GlobalDictionaryEncodedFieldColumnWriter.java
index d9f00bb2321..99f68dbffee 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/GlobalDictionaryEncodedFieldColumnWriter.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/GlobalDictionaryEncodedFieldColumnWriter.java
@@ -40,6 +40,7 @@ import org.apache.druid.segment.data.FixedIndexedIntWriter;
 import org.apache.druid.segment.data.GenericIndexedWriter;
 import org.apache.druid.segment.data.SingleValueColumnarIntsSerializer;
 import org.apache.druid.segment.data.VSizeColumnarIntsSerializer;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.DictionaryEncodedColumnPartSerde;
 import org.apache.druid.segment.serde.Serializer;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
@@ -293,7 +294,7 @@ public abstract class GlobalDictionaryEncodedFieldColumnWriter<T>
         }
       }
     };
-    final String fieldFileName = NestedCommonFormatColumnSerializer.getInternalFileName(columnName, fieldName);
+    final String fieldFileName = ColumnSerializerUtils.getInternalFileName(columnName, fieldName);
     final long size = fieldSerializer.getSerializedSize();
     log.debug("Column [%s] serializing [%s] field of size [%d].", columnName, fieldName, size);
     try (SmooshedWriter smooshChannel = smoosher.addWithSmooshedWriter(fieldFileName, size)) {
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedCommonFormatColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedCommonFormatColumnSerializer.java
index 68e1da96756..e630b64d7d6 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedCommonFormatColumnSerializer.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedCommonFormatColumnSerializer.java
@@ -21,9 +21,9 @@ package org.apache.druid.segment.nested;
 
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.io.smoosh.FileSmoosher;
-import org.apache.druid.java.util.common.io.smoosh.SmooshedWriter;
 import org.apache.druid.segment.GenericColumnSerializer;
 import org.apache.druid.segment.data.VByte;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.Serializer;
 
 import java.io.IOException;
@@ -50,18 +50,7 @@ import java.util.SortedMap;
 public abstract class NestedCommonFormatColumnSerializer implements GenericColumnSerializer<StructuredData>
 {
   public static final byte V0 = 0x00;
-  public static final String STRING_DICTIONARY_FILE_NAME = "__stringDictionary";
-  public static final String LONG_DICTIONARY_FILE_NAME = "__longDictionary";
-  public static final String DOUBLE_DICTIONARY_FILE_NAME = "__doubleDictionary";
-  public static final String ARRAY_DICTIONARY_FILE_NAME = "__arrayDictionary";
-  public static final String ARRAY_ELEMENT_DICTIONARY_FILE_NAME = "__arrayElementDictionary";
-  public static final String ENCODED_VALUE_COLUMN_FILE_NAME = "__encodedColumn";
-  public static final String LONG_VALUE_COLUMN_FILE_NAME = "__longColumn";
-  public static final String DOUBLE_VALUE_COLUMN_FILE_NAME = "__doubleColumn";
-  public static final String BITMAP_INDEX_FILE_NAME = "__valueIndexes";
-  public static final String ARRAY_ELEMENT_BITMAP_INDEX_FILE_NAME = "__arrayElementIndexes";
   public static final String RAW_FILE_NAME = "__raw";
-  public static final String NULL_BITMAP_FILE_NAME = "__nullIndex";
   public static final String NESTED_FIELD_PREFIX = "__field_";
 
   public abstract void openDictionaryWriter() throws IOException;
@@ -86,18 +75,12 @@ public abstract class NestedCommonFormatColumnSerializer implements GenericColum
 
   protected void writeInternal(FileSmoosher smoosher, Serializer serializer, String fileName) throws IOException
   {
-    final String internalName = getInternalFileName(getColumnName(), fileName);
-    try (SmooshedWriter smooshChannel = smoosher.addWithSmooshedWriter(internalName, serializer.getSerializedSize())) {
-      serializer.writeTo(smooshChannel, smoosher);
-    }
+    ColumnSerializerUtils.writeInternal(smoosher, serializer, getColumnName(), fileName);
   }
 
   protected void writeInternal(FileSmoosher smoosher, ByteBuffer buffer, String fileName) throws IOException
   {
-    final String internalName = getInternalFileName(getColumnName(), fileName);
-    try (SmooshedWriter smooshChannel = smoosher.addWithSmooshedWriter(internalName, buffer.capacity())) {
-      smooshChannel.write(buffer);
-    }
+    ColumnSerializerUtils.writeInternal(smoosher, buffer, getColumnName(), fileName);
   }
 
   protected void writeV0Header(WritableByteChannel channel, ByteBuffer columnNameBuffer) throws IOException
@@ -108,8 +91,7 @@ public abstract class NestedCommonFormatColumnSerializer implements GenericColum
 
   protected ByteBuffer computeFilenameBytes()
   {
-    final String columnName = getColumnName();
-    final byte[] bytes = StringUtils.toUtf8(columnName);
+    final byte[] bytes = StringUtils.toUtf8(getColumnName());
     final int length = VByte.computeIntSize(bytes.length);
     final ByteBuffer buffer = ByteBuffer.allocate(length + bytes.length).order(ByteOrder.nativeOrder());
     VByte.writeInt(buffer, bytes.length);
@@ -117,12 +99,4 @@ public abstract class NestedCommonFormatColumnSerializer implements GenericColum
     buffer.flip();
     return buffer;
   }
-
-  /**
-   * Nested field columns are stored in separate
-   */
-  public static String getInternalFileName(String fileNameBase, String field)
-  {
-    return fileNameBase + "." + field;
-  }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java
index bcab3edd889..7dea3b664fd 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java
@@ -34,7 +34,6 @@ import org.apache.druid.java.util.common.io.smoosh.SmooshedWriter;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.math.expr.ExprEval;
 import org.apache.druid.segment.ColumnValueSelector;
-import org.apache.druid.segment.IndexMerger;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.StringEncodingStrategies;
@@ -48,10 +47,10 @@ import org.apache.druid.segment.data.FixedIndexedWriter;
 import org.apache.druid.segment.data.FrontCodedIntArrayIndexedWriter;
 import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.GenericIndexedWriter;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import javax.annotation.Nullable;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -248,7 +247,7 @@ public class NestedDataColumnSerializer extends NestedCommonFormatColumnSerializ
   public void open() throws IOException
   {
     rawWriter = new CompressedVariableSizedBlobColumnSerializer(
-        getInternalFileName(name, RAW_FILE_NAME),
+        ColumnSerializerUtils.getInternalFileName(name, RAW_FILE_NAME),
         segmentWriteOutMedium,
         indexSpec.getJsonCompression() != null ? indexSpec.getJsonCompression() : CompressionStrategy.LZ4
     );
@@ -394,18 +393,6 @@ public class NestedDataColumnSerializer extends NestedCommonFormatColumnSerializ
   {
     if (!closedForWrite) {
       closedForWrite = true;
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      IndexMerger.SERIALIZER_UTILS.writeString(
-          baos,
-          NestedDataComplexTypeSerde.OBJECT_MAPPER.writeValueAsString(
-              new NestedDataColumnMetadata(
-                  ByteOrder.nativeOrder(),
-                  indexSpec.getBitmapSerdeFactory(),
-                  name,
-                  !nullRowsBitmap.isEmpty()
-              )
-          )
-      );
       nullBitmapWriter.write(nullRowsBitmap);
       columnNameBytes = computeFilenameBytes();
     }
@@ -447,26 +434,26 @@ public class NestedDataColumnSerializer extends NestedCommonFormatColumnSerializ
         smoosher.add(internalName, fileMapper.mapFile(internalName));
       }
     } else {
-      writeInternal(smoosher, dictionaryWriter, STRING_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryWriter, ColumnSerializerUtils.STRING_DICTIONARY_FILE_NAME);
     }
     if (globalDictionaryIdLookup.getLongBuffer() != null) {
-      writeInternal(smoosher, globalDictionaryIdLookup.getLongBuffer(), LONG_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, globalDictionaryIdLookup.getLongBuffer(), ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME);
     } else {
-      writeInternal(smoosher, longDictionaryWriter, LONG_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, longDictionaryWriter, ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME);
     }
     if (globalDictionaryIdLookup.getDoubleBuffer() != null) {
-      writeInternal(smoosher, globalDictionaryIdLookup.getDoubleBuffer(), DOUBLE_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, globalDictionaryIdLookup.getDoubleBuffer(), ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME);
     } else {
-      writeInternal(smoosher, doubleDictionaryWriter, DOUBLE_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, doubleDictionaryWriter, ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME);
     }
     if (globalDictionaryIdLookup.getArrayBuffer() != null) {
-      writeInternal(smoosher, globalDictionaryIdLookup.getArrayBuffer(), ARRAY_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, globalDictionaryIdLookup.getArrayBuffer(), ColumnSerializerUtils.ARRAY_DICTIONARY_FILE_NAME);
     } else {
-      writeInternal(smoosher, arrayDictionaryWriter, ARRAY_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, arrayDictionaryWriter, ColumnSerializerUtils.ARRAY_DICTIONARY_FILE_NAME);
     }
     writeInternal(smoosher, rawWriter, RAW_FILE_NAME);
     if (!nullRowsBitmap.isEmpty()) {
-      writeInternal(smoosher, nullBitmapWriter, NULL_BITMAP_FILE_NAME);
+      writeInternal(smoosher, nullBitmapWriter, ColumnSerializerUtils.NULL_BITMAP_FILE_NAME);
     }
 
 
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializerV4.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializerV4.java
index 3d39b31bbbc..bb015ecce92 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializerV4.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializerV4.java
@@ -47,6 +47,8 @@ import org.apache.druid.segment.data.DictionaryWriter;
 import org.apache.druid.segment.data.FixedIndexedWriter;
 import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.GenericIndexedWriter;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
+import org.apache.druid.segment.serde.ComplexColumnMetadata;
 import org.apache.druid.segment.serde.Serializer;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
@@ -181,7 +183,7 @@ public class NestedDataColumnSerializerV4 implements GenericColumnSerializer<Str
     doubleDictionaryWriter.open();
 
     rawWriter = new CompressedVariableSizedBlobColumnSerializer(
-        NestedCommonFormatColumnSerializer.getInternalFileName(name, RAW_FILE_NAME),
+        ColumnSerializerUtils.getInternalFileName(name, RAW_FILE_NAME),
         segmentWriteOutMedium,
         indexSpec.getJsonCompression() != null ? indexSpec.getJsonCompression() : CompressionStrategy.LZ4
     );
@@ -321,8 +323,8 @@ public class NestedDataColumnSerializerV4 implements GenericColumnSerializer<Str
       ByteArrayOutputStream baos = new ByteArrayOutputStream();
       IndexMerger.SERIALIZER_UTILS.writeString(
           baos,
-          NestedDataComplexTypeSerde.OBJECT_MAPPER.writeValueAsString(
-              new NestedDataColumnMetadata(
+          ColumnSerializerUtils.SMILE_MAPPER.writeValueAsString(
+              new ComplexColumnMetadata(
                   ByteOrder.nativeOrder(),
                   indexSpec.getBitmapSerdeFactory(),
                   name,
@@ -389,7 +391,7 @@ public class NestedDataColumnSerializerV4 implements GenericColumnSerializer<Str
 
   private void writeInternal(FileSmoosher smoosher, Serializer serializer, String fileName) throws IOException
   {
-    final String internalName = NestedCommonFormatColumnSerializer.getInternalFileName(name, fileName);
+    final String internalName = ColumnSerializerUtils.getInternalFileName(name, fileName);
     try (SmooshedWriter smooshChannel = smoosher.addWithSmooshedWriter(internalName, serializer.getSerializedSize())) {
       serializer.writeTo(smooshChannel, smoosher);
     }
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplier.java
index f0785d85587..7c5e722ca67 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplier.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplier.java
@@ -38,6 +38,7 @@ import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.VByte;
 import org.apache.druid.segment.index.SimpleImmutableBitmapIndex;
 import org.apache.druid.segment.index.semantic.NullValueIndex;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde;
 
 import javax.annotation.Nullable;
@@ -79,7 +80,7 @@ public class NestedDataColumnSupplier implements Supplier<NestedCommonFormatColu
         final ByteBuffer stringDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.STRING_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.STRING_DICTIONARY_FILE_NAME
         );
 
         stringDictionarySupplier = StringEncodingStrategies.getStringDictionarySupplier(
@@ -91,7 +92,7 @@ public class NestedDataColumnSupplier implements Supplier<NestedCommonFormatColu
         final ByteBuffer longDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME
         );
         longDictionarySupplier = FixedIndexed.read(
             longDictionaryBuffer,
@@ -102,7 +103,7 @@ public class NestedDataColumnSupplier implements Supplier<NestedCommonFormatColu
         final ByteBuffer doubleDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME
         );
         doubleDictionarySupplier = FixedIndexed.read(
             doubleDictionaryBuffer,
@@ -113,7 +114,7 @@ public class NestedDataColumnSupplier implements Supplier<NestedCommonFormatColu
         final ByteBuffer arrayDictionarybuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.ARRAY_DICTIONARY_FILE_NAME
         );
         arrayDictionarySupplier = FrontCodedIntArrayIndexed.read(
             arrayDictionarybuffer,
@@ -125,7 +126,7 @@ public class NestedDataColumnSupplier implements Supplier<NestedCommonFormatColu
             NestedCommonFormatColumnSerializer.RAW_FILE_NAME
         );
         compressedRawColumnSupplier = CompressedVariableSizedBlobColumnSupplier.fromByteBuffer(
-            NestedCommonFormatColumnSerializer.getInternalFileName(
+            ColumnSerializerUtils.getInternalFileName(
                 columnName,
                 NestedCommonFormatColumnSerializer.RAW_FILE_NAME
             ),
@@ -138,7 +139,7 @@ public class NestedDataColumnSupplier implements Supplier<NestedCommonFormatColu
           final ByteBuffer nullIndexBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
               mapper,
               columnName,
-              NestedCommonFormatColumnSerializer.NULL_BITMAP_FILE_NAME
+              ColumnSerializerUtils.NULL_BITMAP_FILE_NAME
           );
           nullValues = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(nullIndexBuffer);
         } else {
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4.java
index 26b34c8c4c2..40eb5a8b142 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4.java
@@ -37,6 +37,8 @@ import org.apache.druid.segment.data.FixedIndexed;
 import org.apache.druid.segment.data.FrontCodedIntArrayIndexed;
 import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.Indexed;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
+import org.apache.druid.segment.serde.ComplexColumnMetadata;
 
 import javax.annotation.Nullable;
 import java.io.IOException;
@@ -77,7 +79,7 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
     if (version == 0x03 || version == 0x04 || version == 0x05) {
       try {
         final SmooshedFileMapper mapper = columnBuilder.getFileMapper();
-        final NestedDataColumnMetadata metadata;
+        final ComplexColumnMetadata metadata;
         final GenericIndexed<String> fields;
         final FieldTypeInfo fieldInfo;
         final CompressedVariableSizedBlobColumnSupplier compressedRawColumnSupplier;
@@ -91,7 +93,7 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
 
         metadata = jsonMapper.readValue(
             IndexMerger.SERIALIZER_UTILS.readString(bb),
-            NestedDataColumnMetadata.class
+            ComplexColumnMetadata.class
         );
         fields = GenericIndexed.read(bb, GenericIndexed.STRING_STRATEGY, mapper);
         fieldInfo = FieldTypeInfo.read(bb, fields.size());
@@ -112,7 +114,7 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
         final ByteBuffer stringDictionaryBuffer = loadInternalFile(
             mapper,
             metadata,
-            NestedCommonFormatColumnSerializer.STRING_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.STRING_DICTIONARY_FILE_NAME
         );
 
         stringDictionarySupplier = StringEncodingStrategies.getStringDictionarySupplier(
@@ -123,7 +125,7 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
         final ByteBuffer longDictionaryBuffer = loadInternalFile(
             mapper,
             metadata,
-            NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME
         );
         longDictionarySupplier = FixedIndexed.read(
             longDictionaryBuffer,
@@ -134,7 +136,7 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
         final ByteBuffer doubleDictionaryBuffer = loadInternalFile(
             mapper,
             metadata,
-            NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME
         );
         doubleDictionarySupplier = FixedIndexed.read(
             doubleDictionaryBuffer,
@@ -146,7 +148,7 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
           final ByteBuffer arrayDictionarybuffer = loadInternalFile(
               mapper,
               metadata,
-              NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME
+              ColumnSerializerUtils.ARRAY_DICTIONARY_FILE_NAME
           );
           arrayDictionarySupplier = FrontCodedIntArrayIndexed.read(
               arrayDictionarybuffer,
@@ -157,7 +159,7 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
         }
         final ByteBuffer rawBuffer = loadInternalFile(mapper, metadata, NestedCommonFormatColumnSerializer.RAW_FILE_NAME);
         compressedRawColumnSupplier = CompressedVariableSizedBlobColumnSupplier.fromByteBuffer(
-            NestedCommonFormatColumnSerializer.getInternalFileName(
+            ColumnSerializerUtils.getInternalFileName(
                 metadata.getFileNameBase(), NestedCommonFormatColumnSerializer.RAW_FILE_NAME
             ),
             rawBuffer,
@@ -169,7 +171,7 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
           final ByteBuffer nullIndexBuffer = loadInternalFile(
               mapper,
               metadata,
-              NestedCommonFormatColumnSerializer.NULL_BITMAP_FILE_NAME
+              ColumnSerializerUtils.NULL_BITMAP_FILE_NAME
           );
           nullValues = metadata.getBitmapSerdeFactory().getObjectStrategy().fromByteBufferWithSize(nullIndexBuffer);
         } else {
@@ -335,12 +337,12 @@ public class NestedDataColumnSupplierV4 implements Supplier<ComplexColumn>
 
   private static ByteBuffer loadInternalFile(
       SmooshedFileMapper fileMapper,
-      NestedDataColumnMetadata metadata,
+      ComplexColumnMetadata metadata,
       String internalFileName
   ) throws IOException
   {
     return fileMapper.mapFile(
-        NestedCommonFormatColumnSerializer.getInternalFileName(metadata.getFileNameBase(), internalFileName)
+        ColumnSerializerUtils.getInternalFileName(metadata.getFileNameBase(), internalFileName)
     );
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV4.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV4.java
index c67b0a00050..124c3fae1a0 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV4.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV4.java
@@ -29,6 +29,7 @@ import org.apache.druid.segment.data.CompressedVariableSizedBlobColumnSupplier;
 import org.apache.druid.segment.data.FixedIndexed;
 import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.Indexed;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -81,7 +82,7 @@ public final class NestedDataColumnV4<TStringDictionary extends Indexed<ByteBuff
   @Override
   public String getFieldFileName(String fileNameBase, String field, int fieldIndex)
   {
-    return NestedCommonFormatColumnSerializer.getInternalFileName(
+    return ColumnSerializerUtils.getInternalFileName(
         fileNameBase,
         NestedCommonFormatColumnSerializer.NESTED_FIELD_PREFIX + fieldIndex
     );
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV5.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV5.java
index 47850afef50..9962f8b4633 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV5.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV5.java
@@ -30,6 +30,7 @@ import org.apache.druid.segment.data.FixedIndexed;
 import org.apache.druid.segment.data.FrontCodedIntArrayIndexed;
 import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.Indexed;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -92,7 +93,7 @@ public class NestedDataColumnV5<TStringDictionary extends Indexed<ByteBuffer>>
   @Override
   public String getFieldFileName(String fileNameBase, String field, int fieldIndex)
   {
-    return NestedCommonFormatColumnSerializer.getInternalFileName(
+    return ColumnSerializerUtils.getInternalFileName(
         fileNameBase,
         NestedCommonFormatColumnSerializer.NESTED_FIELD_PREFIX + fieldIndex
     );
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java
index 56a66c64c06..cb11f62bb0f 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java
@@ -20,13 +20,8 @@
 package org.apache.druid.segment.nested;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
 import it.unimi.dsi.fastutil.Hash;
 import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.guice.BuiltInTypesModule;
-import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.guava.Comparators;
 import org.apache.druid.segment.DimensionHandler;
@@ -41,6 +36,7 @@ import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ObjectStrategyComplexTypeStrategy;
 import org.apache.druid.segment.column.TypeStrategy;
 import org.apache.druid.segment.data.ObjectStrategy;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.ComplexMetricExtractor;
 import org.apache.druid.segment.serde.ComplexMetricSerde;
 
@@ -52,20 +48,8 @@ public class NestedDataComplexTypeSerde extends ComplexMetricSerde
 {
   public static final String TYPE_NAME = "json";
 
-  public static final ObjectMapper OBJECT_MAPPER;
-
   public static final NestedDataComplexTypeSerde INSTANCE = new NestedDataComplexTypeSerde();
 
-  static {
-    final SmileFactory smileFactory = new SmileFactory();
-    smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false);
-    smileFactory.delegateToTextual(true);
-    final ObjectMapper mapper = new DefaultObjectMapper(smileFactory, null);
-    mapper.getFactory().setCodec(mapper);
-    mapper.registerModules(BuiltInTypesModule.getJacksonModulesList());
-    OBJECT_MAPPER = mapper;
-  }
-
   @Override
   public String getTypeName()
   {
@@ -95,7 +79,7 @@ public class NestedDataComplexTypeSerde extends ComplexMetricSerde
         buffer,
         builder,
         columnConfig,
-        OBJECT_MAPPER
+        ColumnSerializerUtils.SMILE_MAPPER
     );
     final ColumnCapabilitiesImpl capabilitiesBuilder = builder.getCapabilitiesBuilder();
     capabilitiesBuilder.setDictionaryEncoded(true);
@@ -136,7 +120,7 @@ public class NestedDataComplexTypeSerde extends ComplexMetricSerde
         final byte[] bytes = new byte[numBytes];
         buffer.get(bytes, 0, numBytes);
         try {
-          return OBJECT_MAPPER.readValue(bytes, StructuredData.class);
+          return ColumnSerializerUtils.SMILE_MAPPER.readValue(bytes, StructuredData.class);
         }
         catch (IOException e) {
           throw new ISE(e, "Unable to deserialize value");
@@ -151,12 +135,18 @@ public class NestedDataComplexTypeSerde extends ComplexMetricSerde
           return new byte[0];
         }
         try {
-          return OBJECT_MAPPER.writeValueAsBytes(val);
+          return ColumnSerializerUtils.SMILE_MAPPER.writeValueAsBytes(val);
         }
         catch (JsonProcessingException e) {
           throw new ISE(e, "Unable to serialize value [%s]", val);
         }
       }
+
+      @Override
+      public boolean readRetainsBufferReference()
+      {
+        return false;
+      }
     };
   }
 
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java
index 1c9d5a6aa35..e535d9cb54b 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java
@@ -70,6 +70,7 @@ import org.apache.druid.segment.index.semantic.NumericRangeIndexes;
 import org.apache.druid.segment.index.semantic.StringValueSetIndexes;
 import org.apache.druid.segment.index.semantic.ValueIndexes;
 import org.apache.druid.segment.index.semantic.ValueSetIndexes;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde;
 
 import javax.annotation.Nonnull;
@@ -105,12 +106,12 @@ public class ScalarDoubleColumnAndIndexSupplier implements Supplier<NestedCommon
         final ByteBuffer doubleDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME
         );
         final ByteBuffer doublesValueColumn = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.DOUBLE_VALUE_COLUMN_FILE_NAME
+            ColumnSerializerUtils.DOUBLE_VALUE_COLUMN_FILE_NAME
         );
 
         final Supplier<FixedIndexed<Double>> doubleDictionarySupplier = FixedIndexed.read(
@@ -127,7 +128,7 @@ public class ScalarDoubleColumnAndIndexSupplier implements Supplier<NestedCommon
         final ByteBuffer valueIndexBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.BITMAP_INDEX_FILE_NAME
+            ColumnSerializerUtils.BITMAP_INDEX_FILE_NAME
         );
         GenericIndexed<ImmutableBitmap> rBitmaps = GenericIndexed.read(
             valueIndexBuffer,
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnSerializer.java
index 874b8b309a4..718981ae19a 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnSerializer.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnSerializer.java
@@ -31,6 +31,7 @@ import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.ColumnarDoublesSerializer;
 import org.apache.druid.segment.data.CompressionFactory;
 import org.apache.druid.segment.data.FixedIndexedWriter;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import javax.annotation.Nullable;
@@ -129,16 +130,16 @@ public class ScalarDoubleColumnSerializer extends ScalarNestedCommonFormatColumn
   @Override
   protected void writeValueColumn(FileSmoosher smoosher) throws IOException
   {
-    writeInternal(smoosher, doublesSerializer, DOUBLE_VALUE_COLUMN_FILE_NAME);
+    writeInternal(smoosher, doublesSerializer, ColumnSerializerUtils.DOUBLE_VALUE_COLUMN_FILE_NAME);
   }
 
   @Override
   protected void writeDictionaryFile(FileSmoosher smoosher) throws IOException
   {
     if (dictionaryIdLookup.getDoubleBuffer() != null) {
-      writeInternal(smoosher, dictionaryIdLookup.getDoubleBuffer(), DOUBLE_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryIdLookup.getDoubleBuffer(), ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME);
     } else {
-      writeInternal(smoosher, dictionaryWriter, DOUBLE_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryWriter, ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME);
     }
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java
index 068613aba68..a9932f7ae3c 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java
@@ -69,6 +69,7 @@ import org.apache.druid.segment.index.semantic.NumericRangeIndexes;
 import org.apache.druid.segment.index.semantic.StringValueSetIndexes;
 import org.apache.druid.segment.index.semantic.ValueIndexes;
 import org.apache.druid.segment.index.semantic.ValueSetIndexes;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde;
 
 import javax.annotation.Nonnull;
@@ -104,17 +105,17 @@ public class ScalarLongColumnAndIndexSupplier implements Supplier<NestedCommonFo
         final ByteBuffer longDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME
         );
         final ByteBuffer longsValueColumn = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.LONG_VALUE_COLUMN_FILE_NAME
+            ColumnSerializerUtils.LONG_VALUE_COLUMN_FILE_NAME
         );
         final ByteBuffer valueIndexBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.BITMAP_INDEX_FILE_NAME
+            ColumnSerializerUtils.BITMAP_INDEX_FILE_NAME
         );
         GenericIndexed<ImmutableBitmap> rBitmaps = GenericIndexed.read(
             valueIndexBuffer,
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnSerializer.java
index 46b70d9907c..00d7c16813d 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnSerializer.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnSerializer.java
@@ -31,6 +31,7 @@ import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.ColumnarLongsSerializer;
 import org.apache.druid.segment.data.CompressionFactory;
 import org.apache.druid.segment.data.FixedIndexedWriter;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import javax.annotation.Nullable;
@@ -129,16 +130,16 @@ public class ScalarLongColumnSerializer extends ScalarNestedCommonFormatColumnSe
   @Override
   protected void writeValueColumn(FileSmoosher smoosher) throws IOException
   {
-    writeInternal(smoosher, longsSerializer, LONG_VALUE_COLUMN_FILE_NAME);
+    writeInternal(smoosher, longsSerializer, ColumnSerializerUtils.LONG_VALUE_COLUMN_FILE_NAME);
   }
 
   @Override
   protected void writeDictionaryFile(FileSmoosher smoosher) throws IOException
   {
     if (dictionaryIdLookup.getLongBuffer() != null) {
-      writeInternal(smoosher, dictionaryIdLookup.getLongBuffer(), LONG_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryIdLookup.getLongBuffer(), ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME);
     } else {
-      writeInternal(smoosher, dictionaryWriter, LONG_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryWriter, ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME);
     }
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java
index 771cdb7fb5b..1010003a611 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java
@@ -36,6 +36,7 @@ import org.apache.druid.segment.data.DictionaryWriter;
 import org.apache.druid.segment.data.FixedIndexedIntWriter;
 import org.apache.druid.segment.data.GenericIndexedWriter;
 import org.apache.druid.segment.data.SingleValueColumnarIntsSerializer;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import javax.annotation.Nullable;
@@ -220,9 +221,9 @@ public abstract class ScalarNestedCommonFormatColumnSerializer<T> extends Nested
 
     writeV0Header(channel, columnNameBytes);
     writeDictionaryFile(smoosher);
-    writeInternal(smoosher, encodedValueSerializer, ENCODED_VALUE_COLUMN_FILE_NAME);
+    writeInternal(smoosher, encodedValueSerializer, ColumnSerializerUtils.ENCODED_VALUE_COLUMN_FILE_NAME);
     writeValueColumn(smoosher);
-    writeInternal(smoosher, bitmapIndexWriter, BITMAP_INDEX_FILE_NAME);
+    writeInternal(smoosher, bitmapIndexWriter, ColumnSerializerUtils.BITMAP_INDEX_FILE_NAME);
 
     log.info("Column [%s] serialized successfully.", name);
   }
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnAndIndexSupplier.java
index 7d957e02a5b..c4850905b7f 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnAndIndexSupplier.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnAndIndexSupplier.java
@@ -34,6 +34,7 @@ import org.apache.druid.segment.data.CompressedVSizeColumnarIntsSupplier;
 import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.Indexed;
 import org.apache.druid.segment.data.VByte;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde;
 import org.apache.druid.segment.serde.StringUtf8ColumnIndexSupplier;
 
@@ -63,7 +64,7 @@ public class ScalarStringColumnAndIndexSupplier implements Supplier<NestedCommon
         final ByteBuffer stringDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.STRING_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.STRING_DICTIONARY_FILE_NAME
         );
 
         dictionarySupplier = StringEncodingStrategies.getStringDictionarySupplier(
@@ -74,7 +75,7 @@ public class ScalarStringColumnAndIndexSupplier implements Supplier<NestedCommon
         final ByteBuffer encodedValueColumn = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.ENCODED_VALUE_COLUMN_FILE_NAME
+            ColumnSerializerUtils.ENCODED_VALUE_COLUMN_FILE_NAME
         );
         final CompressedVSizeColumnarIntsSupplier ints = CompressedVSizeColumnarIntsSupplier.fromByteBuffer(
             encodedValueColumn,
@@ -83,7 +84,7 @@ public class ScalarStringColumnAndIndexSupplier implements Supplier<NestedCommon
         final ByteBuffer valueIndexBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.BITMAP_INDEX_FILE_NAME
+            ColumnSerializerUtils.BITMAP_INDEX_FILE_NAME
         );
         GenericIndexed<ImmutableBitmap> valueIndexes = GenericIndexed.read(
             valueIndexBuffer,
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnSerializer.java
index 7ec884de087..380da1ea982 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnSerializer.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnSerializer.java
@@ -30,6 +30,7 @@ import org.apache.druid.math.expr.ExpressionType;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.StringEncodingStrategies;
 import org.apache.druid.segment.column.StringUtf8DictionaryEncodedColumn;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import javax.annotation.Nullable;
@@ -127,7 +128,7 @@ public class ScalarStringColumnSerializer extends ScalarNestedCommonFormatColumn
         smoosher.add(name, fileMapper.mapFile(name));
       }
     } else {
-      writeInternal(smoosher, dictionaryWriter, STRING_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryWriter, ColumnSerializerUtils.STRING_DICTIONARY_FILE_NAME);
     }
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/StructuredData.java b/processing/src/main/java/org/apache/druid/segment/nested/StructuredData.java
index 9dfa79b57d9..b7a627c389a 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/StructuredData.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/StructuredData.java
@@ -26,6 +26,7 @@ import net.jpountz.xxhash.XXHash64;
 import net.jpountz.xxhash.XXHashFactory;
 import org.apache.druid.java.util.common.guava.Comparators;
 import org.apache.druid.segment.column.TypeStrategies;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 
 import javax.annotation.Nullable;
 import java.util.Arrays;
@@ -45,7 +46,7 @@ public class StructuredData implements Comparable<StructuredData>
   private static long computeHash(StructuredData data)
   {
     try {
-      final byte[] bytes = NestedDataComplexTypeSerde.OBJECT_MAPPER.writeValueAsBytes(data.value);
+      final byte[] bytes = ColumnSerializerUtils.SMILE_MAPPER.writeValueAsBytes(data.value);
       return HASH_FUNCTION.hash(bytes, 0, bytes.length, SEED);
     }
     catch (JsonProcessingException e) {
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnAndIndexSupplier.java
index 7994ae79a0f..aa1dd143e4e 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnAndIndexSupplier.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnAndIndexSupplier.java
@@ -52,6 +52,7 @@ import org.apache.druid.segment.index.SimpleImmutableBitmapIndex;
 import org.apache.druid.segment.index.semantic.ArrayElementIndexes;
 import org.apache.druid.segment.index.semantic.NullValueIndex;
 import org.apache.druid.segment.index.semantic.ValueIndexes;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde;
 
 import javax.annotation.Nonnull;
@@ -97,7 +98,7 @@ public class VariantColumnAndIndexSupplier implements Supplier<NestedCommonForma
         final ByteBuffer stringDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.STRING_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.STRING_DICTIONARY_FILE_NAME
         );
 
         stringDictionarySupplier = StringEncodingStrategies.getStringDictionarySupplier(
@@ -108,7 +109,7 @@ public class VariantColumnAndIndexSupplier implements Supplier<NestedCommonForma
         final ByteBuffer encodedValueColumn = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.ENCODED_VALUE_COLUMN_FILE_NAME
+            ColumnSerializerUtils.ENCODED_VALUE_COLUMN_FILE_NAME
         );
         final CompressedVSizeColumnarIntsSupplier ints = CompressedVSizeColumnarIntsSupplier.fromByteBuffer(
             encodedValueColumn,
@@ -117,22 +118,22 @@ public class VariantColumnAndIndexSupplier implements Supplier<NestedCommonForma
         final ByteBuffer longDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME
         );
         final ByteBuffer doubleDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME
         );
         final ByteBuffer arrayElementDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.ARRAY_ELEMENT_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.ARRAY_ELEMENT_DICTIONARY_FILE_NAME
         );
         final ByteBuffer valueIndexBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.BITMAP_INDEX_FILE_NAME
+            ColumnSerializerUtils.BITMAP_INDEX_FILE_NAME
         );
         final GenericIndexed<ImmutableBitmap> valueIndexes = GenericIndexed.read(
             valueIndexBuffer,
@@ -142,7 +143,7 @@ public class VariantColumnAndIndexSupplier implements Supplier<NestedCommonForma
         final ByteBuffer elementIndexBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.ARRAY_ELEMENT_BITMAP_INDEX_FILE_NAME
+            ColumnSerializerUtils.ARRAY_ELEMENT_BITMAP_INDEX_FILE_NAME
         );
         final GenericIndexed<ImmutableBitmap> arrayElementIndexes = GenericIndexed.read(
             elementIndexBuffer,
@@ -166,7 +167,7 @@ public class VariantColumnAndIndexSupplier implements Supplier<NestedCommonForma
         final ByteBuffer arrayDictionarybuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
             mapper,
             columnName,
-            NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME
+            ColumnSerializerUtils.ARRAY_DICTIONARY_FILE_NAME
         );
         arrayDictionarySupplier = FrontCodedIntArrayIndexed.read(
             arrayDictionarybuffer,
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnSerializer.java
index abd88b57df0..b1454d85004 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnSerializer.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnSerializer.java
@@ -48,6 +48,7 @@ import org.apache.druid.segment.data.FixedIndexedWriter;
 import org.apache.druid.segment.data.FrontCodedIntArrayIndexedWriter;
 import org.apache.druid.segment.data.GenericIndexedWriter;
 import org.apache.druid.segment.data.SingleValueColumnarIntsSerializer;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import javax.annotation.Nullable;
@@ -413,28 +414,28 @@ public class VariantColumnSerializer extends NestedCommonFormatColumnSerializer
         smoosher.add(internalName, fileMapper.mapFile(internalName));
       }
     } else {
-      writeInternal(smoosher, dictionaryWriter, STRING_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryWriter, ColumnSerializerUtils.STRING_DICTIONARY_FILE_NAME);
     }
     if (dictionaryIdLookup.getLongBuffer() != null) {
-      writeInternal(smoosher, dictionaryIdLookup.getLongBuffer(), LONG_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryIdLookup.getLongBuffer(), ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME);
     } else {
-      writeInternal(smoosher, longDictionaryWriter, LONG_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, longDictionaryWriter, ColumnSerializerUtils.LONG_DICTIONARY_FILE_NAME);
     }
     if (dictionaryIdLookup.getDoubleBuffer() != null) {
-      writeInternal(smoosher, dictionaryIdLookup.getDoubleBuffer(), DOUBLE_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryIdLookup.getDoubleBuffer(), ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME);
     } else {
-      writeInternal(smoosher, doubleDictionaryWriter, DOUBLE_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, doubleDictionaryWriter, ColumnSerializerUtils.DOUBLE_DICTIONARY_FILE_NAME);
     }
     if (dictionaryIdLookup.getArrayBuffer() != null) {
-      writeInternal(smoosher, dictionaryIdLookup.getArrayBuffer(), ARRAY_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, dictionaryIdLookup.getArrayBuffer(), ColumnSerializerUtils.ARRAY_DICTIONARY_FILE_NAME);
     } else {
-      writeInternal(smoosher, arrayDictionaryWriter, ARRAY_DICTIONARY_FILE_NAME);
+      writeInternal(smoosher, arrayDictionaryWriter, ColumnSerializerUtils.ARRAY_DICTIONARY_FILE_NAME);
     }
 
-    writeInternal(smoosher, arrayElementDictionaryWriter, ARRAY_ELEMENT_DICTIONARY_FILE_NAME);
-    writeInternal(smoosher, encodedValueSerializer, ENCODED_VALUE_COLUMN_FILE_NAME);
-    writeInternal(smoosher, bitmapIndexWriter, BITMAP_INDEX_FILE_NAME);
-    writeInternal(smoosher, arrayElementIndexWriter, ARRAY_ELEMENT_BITMAP_INDEX_FILE_NAME);
+    writeInternal(smoosher, arrayElementDictionaryWriter, ColumnSerializerUtils.ARRAY_ELEMENT_DICTIONARY_FILE_NAME);
+    writeInternal(smoosher, encodedValueSerializer, ColumnSerializerUtils.ENCODED_VALUE_COLUMN_FILE_NAME);
+    writeInternal(smoosher, bitmapIndexWriter, ColumnSerializerUtils.BITMAP_INDEX_FILE_NAME);
+    writeInternal(smoosher, arrayElementIndexWriter, ColumnSerializerUtils.ARRAY_ELEMENT_BITMAP_INDEX_FILE_NAME);
 
     log.info("Column [%s] serialized successfully.", name);
   }
diff --git a/processing/src/main/java/org/apache/druid/segment/serde/ColumnSerializerUtils.java b/processing/src/main/java/org/apache/druid/segment/serde/ColumnSerializerUtils.java
new file mode 100644
index 00000000000..8396f07fd35
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/serde/ColumnSerializerUtils.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.serde;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import org.apache.druid.guice.BuiltInTypesModule;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.io.smoosh.FileSmoosher;
+import org.apache.druid.java.util.common.io.smoosh.SmooshedWriter;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class ColumnSerializerUtils
+{
+  public static final String STRING_DICTIONARY_FILE_NAME = "__stringDictionary";
+  public static final String LONG_DICTIONARY_FILE_NAME = "__longDictionary";
+  public static final String DOUBLE_DICTIONARY_FILE_NAME = "__doubleDictionary";
+  public static final String ARRAY_DICTIONARY_FILE_NAME = "__arrayDictionary";
+  public static final String ARRAY_ELEMENT_DICTIONARY_FILE_NAME = "__arrayElementDictionary";
+  public static final String ENCODED_VALUE_COLUMN_FILE_NAME = "__encodedColumn";
+  public static final String LONG_VALUE_COLUMN_FILE_NAME = "__longColumn";
+  public static final String DOUBLE_VALUE_COLUMN_FILE_NAME = "__doubleColumn";
+  public static final String BITMAP_INDEX_FILE_NAME = "__valueIndexes";
+  public static final String ARRAY_ELEMENT_BITMAP_INDEX_FILE_NAME = "__arrayElementIndexes";
+  public static final String NULL_BITMAP_FILE_NAME = "__nullIndex";
+
+  public static final ObjectMapper SMILE_MAPPER;
+
+  static {
+    final SmileFactory smileFactory = new SmileFactory();
+    smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false);
+    smileFactory.delegateToTextual(true);
+    final ObjectMapper mapper = new DefaultObjectMapper(smileFactory, null);
+    mapper.getFactory().setCodec(mapper);
+    mapper.registerModules(BuiltInTypesModule.getJacksonModulesList());
+    SMILE_MAPPER = mapper;
+  }
+
+  public static void writeInternal(FileSmoosher smoosher, Serializer serializer, String columnName, String fileName)
+      throws IOException
+  {
+    final String internalName = getInternalFileName(columnName, fileName);
+    try (SmooshedWriter smooshChannel = smoosher.addWithSmooshedWriter(internalName, serializer.getSerializedSize())) {
+      serializer.writeTo(smooshChannel, smoosher);
+    }
+  }
+
+  public static void writeInternal(FileSmoosher smoosher, ByteBuffer buffer, String columnName, String fileName)
+      throws IOException
+  {
+    final String internalName = getInternalFileName(columnName, fileName);
+    try (SmooshedWriter smooshChannel = smoosher.addWithSmooshedWriter(internalName, buffer.capacity())) {
+      smooshChannel.write(buffer);
+    }
+  }
+
+  public static String getInternalFileName(String fileNameBase, String field)
+  {
+    return fileNameBase + "." + field;
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnMetadata.java b/processing/src/main/java/org/apache/druid/segment/serde/ComplexColumnMetadata.java
similarity index 94%
rename from processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnMetadata.java
rename to processing/src/main/java/org/apache/druid/segment/serde/ComplexColumnMetadata.java
index 99eced8b087..504ecc5abe6 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnMetadata.java
+++ b/processing/src/main/java/org/apache/druid/segment/serde/ComplexColumnMetadata.java
@@ -18,7 +18,7 @@
  */
 
 
-package org.apache.druid.segment.nested;
+package org.apache.druid.segment.serde;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -26,7 +26,7 @@ import org.apache.druid.segment.data.BitmapSerdeFactory;
 
 import java.nio.ByteOrder;
 
-public class NestedDataColumnMetadata
+public class ComplexColumnMetadata
 {
   private final ByteOrder byteOrder;
   private final BitmapSerdeFactory bitmapSerdeFactory;
@@ -34,7 +34,7 @@ public class NestedDataColumnMetadata
   private final Boolean hasNulls;
 
   @JsonCreator
-  public NestedDataColumnMetadata(
+  public ComplexColumnMetadata(
       @JsonProperty("byteOrder") ByteOrder byteOrder,
       @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory,
       @JsonProperty("fileNameBase") String fileNameBase,
diff --git a/processing/src/main/java/org/apache/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/ComplexMetricSerde.java
index 694009ca8fd..29150530c58 100644
--- a/processing/src/main/java/org/apache/druid/segment/serde/ComplexMetricSerde.java
+++ b/processing/src/main/java/org/apache/druid/segment/serde/ComplexMetricSerde.java
@@ -23,11 +23,14 @@ import com.google.common.base.Function;
 import it.unimi.dsi.fastutil.bytes.ByteArrays;
 import org.apache.druid.guice.annotations.ExtensionPoint;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnConfig;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ObjectStrategyComplexTypeStrategy;
 import org.apache.druid.segment.column.TypeStrategy;
+import org.apache.druid.segment.data.CompressionStrategy;
+import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
 import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
@@ -43,31 +46,6 @@ public abstract class ComplexMetricSerde
 
   public abstract ComplexMetricExtractor getExtractor();
 
-  /**
-   * Deserializes a ByteBuffer and adds it to the ColumnBuilder.  This method allows for the ComplexMetricSerde
-   * to implement it's own versioning scheme to allow for changes of binary format in a forward-compatible manner.
-   *
-   * @param buffer  the buffer to deserialize
-   * @param builder ColumnBuilder to add the column to
-   * @param columnConfig ColumnConfiguration used during deserialization
-   */
-  public void deserializeColumn(
-      ByteBuffer buffer,
-      ColumnBuilder builder,
-      @SuppressWarnings("unused") ColumnConfig columnConfig
-  )
-  {
-    deserializeColumn(buffer, builder);
-  }
-
-
-  /**
-   * {@link ComplexMetricSerde#deserializeColumn(ByteBuffer, ColumnBuilder, ColumnConfig)} should be used instead of this.
-   * This method is left for backward compatibility.
-   */
-  @Deprecated
-  public abstract void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder);
-
   /**
    * This is deprecated because its usage is going to be removed from the code.
    * <p>
@@ -142,14 +120,100 @@ public abstract class ComplexMetricSerde
   }
 
   /**
-   * This method provides the ability for a ComplexMetricSerde to control its own serialization.
-   * For large column (i.e columns greater than {@link Integer#MAX_VALUE}) use
-   * {@link LargeColumnSupportedComplexColumnSerializer}
+   * Deserializes a ByteBuffer and adds it to the ColumnBuilder.  This method allows for the ComplexMetricSerde
+   * to implement it's own versioning scheme to allow for changes of binary format in a forward-compatible manner.
    *
-   * @return an instance of GenericColumnSerializer used for serialization.
+   * @param buffer  the buffer to deserialize
+   * @param builder ColumnBuilder to add the column to
+   * @param columnConfig ColumnConfiguration used during deserialization
    */
+  public void deserializeColumn(
+      ByteBuffer buffer,
+      ColumnBuilder builder,
+      ColumnConfig columnConfig
+  )
+  {
+    deserializeColumn(buffer, builder);
+  }
+
+
+  /**
+   * {@link ComplexMetricSerde#deserializeColumn(ByteBuffer, ColumnBuilder, ColumnConfig)} should be used instead of this.
+   * This method is left for backward compatibility.
+   */
+  @Deprecated
+  public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder)
+  {
+    // default implementation to match default serializer implementation
+    final int position = buffer.position();
+    final byte version = buffer.get();
+    if (version == CompressedComplexColumnSerializer.IS_COMPRESSED) {
+      CompressedComplexColumnSupplier supplier = CompressedComplexColumnSupplier.read(
+          buffer,
+          builder,
+          getTypeName(),
+          getObjectStrategy()
+      );
+      builder.setComplexColumnSupplier(supplier);
+      builder.setNullValueIndexSupplier(supplier.getNullValues());
+      builder.setHasNulls(!supplier.getNullValues().isEmpty());
+    } else {
+      buffer.position(position);
+      builder.setComplexColumnSupplier(
+          new ComplexColumnPartSupplier(
+              getTypeName(),
+              GenericIndexed.read(buffer, getObjectStrategy(), builder.getFileMapper())
+          )
+      );
+    }
+  }
+
+  /**
+   * {@link ComplexMetricSerde#getSerializer(SegmentWriteOutMedium, String, IndexSpec)} should be used instead of this.
+   * This method is left for backward compatibility.
+   */
+  @Nullable
+  @Deprecated
   public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column)
   {
-    return ComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy());
+    return null;
+  }
+
+  /**
+   * This method provides the ability for a ComplexMetricSerde to control its own serialization.
+   * Default implementation uses {@link CompressedComplexColumnSerializer} if {@link IndexSpec#complexMetricCompression}
+   * is not null or uncompressed/none, or {@link LargeColumnSupportedComplexColumnSerializer} if no compression is
+   * specified.
+   *
+   * @return an instance of {@link GenericColumnSerializer} used for serialization.
+   */
+  public GenericColumnSerializer getSerializer(
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      String column,
+      IndexSpec indexSpec
+  )
+  {
+    // backwards compatibility, if defined use it
+    final GenericColumnSerializer serializer = getSerializer(segmentWriteOutMedium, column);
+    if (serializer != null) {
+      return serializer;
+    }
+
+    // otherwise, use compressed or generic indexed based serializer
+    CompressionStrategy strategy = indexSpec.getComplexMetricCompression();
+    if (strategy == null || CompressionStrategy.NONE == strategy || CompressionStrategy.UNCOMPRESSED == strategy) {
+      return LargeColumnSupportedComplexColumnSerializer.create(
+          segmentWriteOutMedium,
+          column,
+          getObjectStrategy()
+      );
+    } else {
+      return CompressedComplexColumnSerializer.create(
+          segmentWriteOutMedium,
+          column,
+          indexSpec,
+          getObjectStrategy()
+      );
+    }
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumn.java b/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumn.java
new file mode 100644
index 00000000000..0811dd22ccf
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumn.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.serde;
+
+import org.apache.druid.collections.bitmap.ImmutableBitmap;
+import org.apache.druid.segment.column.ComplexColumn;
+import org.apache.druid.segment.data.CompressedVariableSizedBlobColumn;
+import org.apache.druid.segment.data.ObjectStrategy;
+import org.apache.druid.utils.CloseableUtils;
+
+import javax.annotation.Nullable;
+import java.nio.ByteBuffer;
+
+public final class CompressedComplexColumn implements ComplexColumn
+{
+  private final String typeName;
+  private final CompressedVariableSizedBlobColumn compressedColumn;
+  private final ImmutableBitmap nullValues;
+  private final ObjectStrategy<?> objectStrategy;
+
+  public CompressedComplexColumn(
+      String typeName,
+      CompressedVariableSizedBlobColumn compressedColumn,
+      ImmutableBitmap nullValues,
+      ObjectStrategy<?> objectStrategy
+  )
+  {
+    this.typeName = typeName;
+    this.compressedColumn = compressedColumn;
+    this.nullValues = nullValues;
+    this.objectStrategy = objectStrategy;
+  }
+
+  @Override
+  public Class<?> getClazz()
+  {
+    return objectStrategy.getClazz();
+  }
+
+  @Override
+  public String getTypeName()
+  {
+    return typeName;
+  }
+
+  @Override
+  @Nullable
+  public Object getRowValue(int rowNum)
+  {
+    if (nullValues.get(rowNum)) {
+      return null;
+    }
+
+    final ByteBuffer valueBuffer = compressedColumn.get(rowNum);
+    return objectStrategy.fromByteBuffer(valueBuffer, valueBuffer.remaining());
+  }
+
+  @Override
+  public int getLength()
+  {
+    return compressedColumn.size();
+  }
+
+  @Override
+  public void close()
+  {
+    CloseableUtils.closeAndWrapExceptions(compressedColumn);
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumnSerializer.java
new file mode 100644
index 00000000000..321cf8bc8b0
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumnSerializer.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.serde;
+
+import com.google.common.base.Preconditions;
+import org.apache.druid.collections.bitmap.ImmutableBitmap;
+import org.apache.druid.collections.bitmap.MutableBitmap;
+import org.apache.druid.java.util.common.io.smoosh.FileSmoosher;
+import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexMerger;
+import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.data.ByteBufferWriter;
+import org.apache.druid.segment.data.CompressedVariableSizedBlobColumnSerializer;
+import org.apache.druid.segment.data.ObjectStrategy;
+import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.WritableByteChannel;
+
+public class CompressedComplexColumnSerializer<T> implements GenericColumnSerializer<T>
+{
+  public static final byte IS_COMPRESSED = Byte.MAX_VALUE;
+  public static final byte V0 = 0x00;
+  public static final String FILE_NAME = "__complexColumn";
+
+  public static GenericColumnSerializer create(
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      String column,
+      IndexSpec indexSpec,
+      ObjectStrategy objectStrategy
+  )
+  {
+    return new CompressedComplexColumnSerializer(column, segmentWriteOutMedium, indexSpec, objectStrategy);
+  }
+
+  public CompressedComplexColumnSerializer(
+      String name,
+      SegmentWriteOutMedium segmentWriteOutMedium,
+      IndexSpec indexSpec,
+      ObjectStrategy<T> strategy
+  )
+  {
+    this.name = name;
+    this.segmentWriteOutMedium = segmentWriteOutMedium;
+    this.indexSpec = indexSpec;
+    this.strategy = strategy;
+  }
+
+  private final String name;
+  private final SegmentWriteOutMedium segmentWriteOutMedium;
+  private final IndexSpec indexSpec;
+  private final ObjectStrategy<T> strategy;
+  private CompressedVariableSizedBlobColumnSerializer writer;
+  private ByteBufferWriter<ImmutableBitmap> nullBitmapWriter;
+  private MutableBitmap nullRowsBitmap;
+
+  private int rowCount = 0;
+  private boolean closedForWrite = false;
+  private byte[] metadataBytes;
+
+  @Override
+  public void open() throws IOException
+  {
+    writer = new CompressedVariableSizedBlobColumnSerializer(
+        ColumnSerializerUtils.getInternalFileName(name, FILE_NAME),
+        segmentWriteOutMedium,
+        indexSpec.getComplexMetricCompression()
+    );
+    writer.open();
+
+    nullBitmapWriter = new ByteBufferWriter<>(
+        segmentWriteOutMedium,
+        indexSpec.getBitmapSerdeFactory().getObjectStrategy()
+    );
+    nullBitmapWriter.open();
+
+    nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap();
+  }
+
+  @Override
+  public void serialize(ColumnValueSelector<? extends T> selector) throws IOException
+  {
+    final T data = selector.getObject();
+    if (data == null) {
+      nullRowsBitmap.add(rowCount);
+    }
+    rowCount++;
+    final byte[] bytes = strategy.toBytes(data);
+    writer.addValue(bytes);
+  }
+
+  @Override
+  public long getSerializedSize() throws IOException
+  {
+    closeForWrite();
+    // COMPRESSED_BYTE + V0 + metadata
+    return 1 + 1 + metadataBytes.length;
+  }
+
+  @Override
+  public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException
+  {
+    Preconditions.checkState(closedForWrite, "Not closed yet!");
+
+    channel.write(ByteBuffer.wrap(new byte[]{IS_COMPRESSED}));
+    channel.write(ByteBuffer.wrap(new byte[]{V0}));
+    channel.write(ByteBuffer.wrap(metadataBytes));
+
+    ColumnSerializerUtils.writeInternal(smoosher, writer, name, FILE_NAME);
+    if (!nullRowsBitmap.isEmpty()) {
+      ColumnSerializerUtils.writeInternal(
+          smoosher,
+          nullBitmapWriter,
+          name,
+          ColumnSerializerUtils.NULL_BITMAP_FILE_NAME
+      );
+    }
+  }
+
+  private void closeForWrite() throws IOException
+  {
+    if (!closedForWrite) {
+      closedForWrite = true;
+      nullBitmapWriter.write(nullRowsBitmap);
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      IndexMerger.SERIALIZER_UTILS.writeString(
+          baos,
+          ColumnSerializerUtils.SMILE_MAPPER.writeValueAsString(
+              new ComplexColumnMetadata(
+                  ByteOrder.nativeOrder(),
+                  indexSpec.getBitmapSerdeFactory(),
+                  name,
+                  !nullRowsBitmap.isEmpty()
+              )
+          )
+      );
+      this.metadataBytes = baos.toByteArray();
+    }
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumnSupplier.java b/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumnSupplier.java
new file mode 100644
index 00000000000..b49c655ff14
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/segment/serde/CompressedComplexColumnSupplier.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.serde;
+
+import com.google.common.base.Supplier;
+import org.apache.druid.collections.bitmap.ImmutableBitmap;
+import org.apache.druid.java.util.common.RE;
+import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper;
+import org.apache.druid.segment.IndexMerger;
+import org.apache.druid.segment.column.ColumnBuilder;
+import org.apache.druid.segment.column.ComplexColumn;
+import org.apache.druid.segment.data.CompressedVariableSizedBlobColumnSupplier;
+import org.apache.druid.segment.data.ObjectStrategy;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class CompressedComplexColumnSupplier implements Supplier<ComplexColumn>
+{
+  public static CompressedComplexColumnSupplier read(
+      ByteBuffer bb,
+      ColumnBuilder columnBuilder,
+      String typeName,
+      ObjectStrategy objectStrategy
+  )
+  {
+    final byte version = bb.get();
+
+    if (version == CompressedComplexColumnSerializer.V0) {
+      try {
+        final ComplexColumnMetadata metadata = ColumnSerializerUtils.SMILE_MAPPER.readValue(
+            IndexMerger.SERIALIZER_UTILS.readString(bb),
+            ComplexColumnMetadata.class
+        );
+        final SmooshedFileMapper mapper = columnBuilder.getFileMapper();
+
+        final CompressedVariableSizedBlobColumnSupplier compressedColumnSupplier;
+        final ImmutableBitmap nullValues;
+
+        final ByteBuffer fileBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
+            mapper,
+            metadata.getFileNameBase(),
+            CompressedComplexColumnSerializer.FILE_NAME
+        );
+
+        compressedColumnSupplier = CompressedVariableSizedBlobColumnSupplier.fromByteBuffer(
+            ColumnSerializerUtils.getInternalFileName(
+                metadata.getFileNameBase(),
+                CompressedComplexColumnSerializer.FILE_NAME
+            ),
+            fileBuffer,
+            metadata.getByteOrder(),
+            objectStrategy.readRetainsBufferReference(),
+            mapper
+        );
+
+        if (metadata.hasNulls()) {
+          columnBuilder.setHasNulls(true);
+          final ByteBuffer nullIndexBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile(
+              mapper,
+              metadata.getFileNameBase(),
+              ColumnSerializerUtils.NULL_BITMAP_FILE_NAME
+          );
+          nullValues = metadata.getBitmapSerdeFactory().getObjectStrategy().fromByteBufferWithSize(nullIndexBuffer);
+        } else {
+          nullValues = metadata.getBitmapSerdeFactory().getBitmapFactory().makeEmptyImmutableBitmap();
+        }
+
+        return new CompressedComplexColumnSupplier(typeName, objectStrategy, compressedColumnSupplier, nullValues);
+      }
+      catch (IOException ex) {
+        throw new RE(ex, "Failed to deserialize V%s column.", version);
+      }
+    }
+    throw new RE("Unknown version " + version);
+  }
+
+  private final String typeName;
+  private final ObjectStrategy objectStrategy;
+  private final CompressedVariableSizedBlobColumnSupplier compressedColumnSupplier;
+  private final ImmutableBitmap nullValues;
+
+  private CompressedComplexColumnSupplier(
+      String typeName,
+      ObjectStrategy objectStrategy,
+      CompressedVariableSizedBlobColumnSupplier compressedColumnSupplier,
+      ImmutableBitmap nullValues
+  )
+  {
+    this.typeName = typeName;
+    this.objectStrategy = objectStrategy;
+    this.compressedColumnSupplier = compressedColumnSupplier;
+    this.nullValues = nullValues;
+  }
+
+  @Override
+  public ComplexColumn get()
+  {
+    return new CompressedComplexColumn(typeName, compressedColumnSupplier.get(), nullValues, objectStrategy);
+  }
+
+  public ImmutableBitmap getNullValues()
+  {
+    return nullValues;
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java
index 6bf897cd6b2..e09855cdab2 100644
--- a/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java
+++ b/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java
@@ -32,7 +32,6 @@ import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.data.BitmapSerdeFactory;
 import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
 import org.apache.druid.segment.nested.NestedCommonFormatColumn;
-import org.apache.druid.segment.nested.NestedCommonFormatColumnSerializer;
 import org.apache.druid.segment.nested.NestedDataColumnSupplier;
 import org.apache.druid.segment.nested.ScalarDoubleColumnAndIndexSupplier;
 import org.apache.druid.segment.nested.ScalarLongColumnAndIndexSupplier;
@@ -70,7 +69,7 @@ public class NestedCommonFormatColumnPartSerde implements ColumnPartSerde
   ) throws IOException
   {
     return fileMapper.mapFile(
-        NestedCommonFormatColumnSerializer.getInternalFileName(fileNameBase, internalFileName)
+        ColumnSerializerUtils.getInternalFileName(fileNameBase, internalFileName)
     );
   }
 
diff --git a/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java
index 270a335bc77..fdeb42ee1ca 100644
--- a/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java
+++ b/processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java
@@ -445,6 +445,10 @@ public class QueryRunnerTestHelper
             maker.apply(
                 "frontCodedMMappedTestIndex",
                 new QueryableIndexSegment(TestIndex.getFrontCodedMMappedTestIndex(), SEGMENT_ID)
+            ),
+            maker.apply(
+                "mMappedTestIndexCompressedComplex",
+                new QueryableIndexSegment(TestIndex.getMMappedTestIndexCompressedComplex(), SEGMENT_ID)
             )
         )
     );
diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java
index 1f5900ce1c9..8dc5120958f 100644
--- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java
+++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ComplexColumn;
@@ -147,7 +148,7 @@ public class SerializablePairLongDoubleComplexMetricSerdeTest
     SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium();
     ByteBuffer compressedBuffer = serializeAllValuesToByteBuffer(
         expected,
-        COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used"),
+        COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used", IndexSpec.DEFAULT),
         expectedCompressedSize
     ).asReadOnlyBuffer();
 
diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java
index 7fc270dae1f..1ca8c44d7be 100644
--- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java
+++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ComplexColumn;
@@ -147,7 +148,7 @@ public class SerializablePairLongFloatComplexMetricSerdeTest
     SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium();
     ByteBuffer compressedBuffer = serializeAllValuesToByteBuffer(
         expected,
-        COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used"),
+        COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used", IndexSpec.DEFAULT),
         expectedCompressedSize
     ).asReadOnlyBuffer();
 
diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java
index 68429ad99f4..3e03e05ee3b 100644
--- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java
+++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ComplexColumn;
@@ -147,7 +148,7 @@ public class SerializablePairLongLongComplexMetricSerdeTest
     SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium();
     ByteBuffer compressedBuffer = serializeAllValuesToByteBuffer(
         expected,
-        COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used"),
+        COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used", IndexSpec.DEFAULT),
         expectedCompressedSize
     ).asReadOnlyBuffer();
 
diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerdeTest.java
index f503874a4d6..b6ba96c1cec 100644
--- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerdeTest.java
+++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexMetricSerdeTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.segment.ColumnValueSelector;
 import org.apache.druid.segment.GenericColumnSerializer;
+import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ComplexColumn;
@@ -152,12 +153,12 @@ public class SerializablePairLongStringComplexMetricSerdeTest
     SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium();
     ByteBuffer legacyBuffer = serializeAllValuesToByteBuffer(
         expected,
-        LEGACY_SERDE.getSerializer(writeOutMedium, "not-used"),
+        LEGACY_SERDE.getSerializer(writeOutMedium, "not-used", IndexSpec.DEFAULT),
         expectedLegacySize
     ).asReadOnlyBuffer();
     ByteBuffer compressedBuffer = serializeAllValuesToByteBuffer(
         expected,
-        COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used"),
+        COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used", IndexSpec.DEFAULT),
         expectedCompressedSize
     ).asReadOnlyBuffer();
 
diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java
index 1657a77b209..d0c93a270d0 100644
--- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java
+++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java
@@ -51,6 +51,7 @@ import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde;
 import org.apache.druid.query.expression.TestExprMacroTable;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.StringEncodingStrategy;
+import org.apache.druid.segment.data.CompressionStrategy;
 import org.apache.druid.segment.data.FrontCodedIndexed;
 import org.apache.druid.segment.incremental.IncrementalIndex;
 import org.apache.druid.segment.incremental.IncrementalIndexSchema;
@@ -195,6 +196,13 @@ public class TestIndex
   private static Supplier<QueryableIndex> mmappedIndex = Suppliers.memoize(
       () -> persistRealtimeAndLoadMMapped(realtimeIndex.get())
   );
+
+  private static Supplier<QueryableIndex> mmappedIndexCompressedComplex = Suppliers.memoize(
+      () -> persistRealtimeAndLoadMMapped(
+          realtimeIndex.get(),
+          IndexSpec.builder().withComplexMetricCompression(CompressionStrategy.LZ4).build()
+      )
+  );
   private static Supplier<QueryableIndex> nonTimeOrderedMmappedIndex = Suppliers.memoize(
       () -> persistRealtimeAndLoadMMapped(nonTimeOrderedRealtimeIndex.get())
   );
@@ -324,6 +332,11 @@ public class TestIndex
     return frontCodedMmappedIndex.get();
   }
 
+  public static QueryableIndex getMMappedTestIndexCompressedComplex()
+  {
+    return mmappedIndexCompressedComplex.get();
+  }
+
   public static IncrementalIndex makeRealtimeIndex(final String resourceFilename)
   {
     return makeRealtimeIndex(resourceFilename, true);
diff --git a/processing/src/test/java/org/apache/druid/segment/data/CompressedVariableSizeBlobColumnTest.java b/processing/src/test/java/org/apache/druid/segment/data/CompressedVariableSizeBlobColumnTest.java
index 11609ffdc97..eaa447a2812 100644
--- a/processing/src/test/java/org/apache/druid/segment/data/CompressedVariableSizeBlobColumnTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/data/CompressedVariableSizeBlobColumnTest.java
@@ -92,7 +92,7 @@ public class CompressedVariableSizeBlobColumnTest
     ).get();
     for (int row = 0; row < numWritten; row++) {
       ByteBuffer value = column.get(row);
-      byte[] bytes = new byte[value.limit()];
+      byte[] bytes = new byte[value.remaining()];
       value.get(bytes);
       Assert.assertArrayEquals("Row " + row, values.get(row), bytes);
     }
diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java
index c7b1facec56..33eaa4a0312 100644
--- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java
+++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java
@@ -61,6 +61,7 @@ import org.apache.druid.segment.index.semantic.DruidPredicateIndexes;
 import org.apache.druid.segment.index.semantic.NullValueIndex;
 import org.apache.druid.segment.index.semantic.StringValueSetIndexes;
 import org.apache.druid.segment.serde.ColumnPartSerde;
+import org.apache.druid.segment.serde.ColumnSerializerUtils;
 import org.apache.druid.segment.serde.ComplexColumnPartSerde;
 import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory;
 import org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory;
@@ -247,7 +248,7 @@ public class NestedDataColumnSupplierV4Test extends InitializedNullHandlingTest
         baseBuffer,
         bob,
         ColumnConfig.SELECTION_SIZE,
-        NestedDataComplexTypeSerde.OBJECT_MAPPER
+        ColumnSerializerUtils.SMILE_MAPPER
     );
     final String expectedReason = "none";
     final AtomicReference<String> failureReason = new AtomicReference<>(expectedReason);
diff --git a/processing/src/test/java/org/apache/druid/segment/serde/HyperUniquesSerdeForTest.java b/processing/src/test/java/org/apache/druid/segment/serde/HyperUniquesSerdeForTest.java
index 5bd56cf7e78..cdc502c6c19 100644
--- a/processing/src/test/java/org/apache/druid/segment/serde/HyperUniquesSerdeForTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/serde/HyperUniquesSerdeForTest.java
@@ -24,11 +24,9 @@ import it.unimi.dsi.fastutil.bytes.ByteArrays;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.hll.HyperLogLogCollector;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.segment.GenericColumnSerializer;
 import org.apache.druid.segment.column.ColumnBuilder;
 import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.ObjectStrategy;
-import org.apache.druid.segment.writeout.SegmentWriteOutMedium;
 
 import java.nio.ByteBuffer;
 import java.util.Comparator;
@@ -141,15 +139,4 @@ public class HyperUniquesSerdeForTest extends ComplexMetricSerde
       }
     };
   }
-
-  @Override
-  public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String metric)
-  {
-    return LargeColumnSupportedComplexColumnSerializer.createWithColumnSize(
-        segmentWriteOutMedium,
-        metric,
-        this.getObjectStrategy(),
-        Integer.MAX_VALUE
-    );
-  }
 }
diff --git a/website/.spelling b/website/.spelling
index 74f660c2213..f8e319d0782 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -299,6 +299,7 @@ colocation
 colocating
 compactable
 compactionTask
+complexMetricCompression
 config
 configs
 consumerProperties

From 0caf383102ce55abb62427975eca3733717a9eff Mon Sep 17 00:00:00 2001
From: Pranav <pranavbhole@gmail.com>
Date: Tue, 27 Aug 2024 00:36:29 -0700
Subject: [PATCH 60/99] Fix buffer capacity race condition in spatial (#16931)

---
 .../druid/collections/spatial/ImmutableFloatNode.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/processing/src/main/java/org/apache/druid/collections/spatial/ImmutableFloatNode.java b/processing/src/main/java/org/apache/druid/collections/spatial/ImmutableFloatNode.java
index 009cc13d91b..8025f83bee3 100644
--- a/processing/src/main/java/org/apache/druid/collections/spatial/ImmutableFloatNode.java
+++ b/processing/src/main/java/org/apache/druid/collections/spatial/ImmutableFloatNode.java
@@ -149,10 +149,10 @@ public class ImmutableFloatNode implements ImmutableNode<float[]>
   {
     final int sizePosition = initialOffset + offsetFromInitial + HEADER_NUM_BYTES + 2 * numDims * Float.BYTES;
     int numBytes = data.getInt(sizePosition);
-    data.position(sizePosition + Integer.BYTES);
-    ByteBuffer tmpBuffer = data.slice();
-    tmpBuffer.limit(numBytes);
-    return bitmapFactory.mapImmutableBitmap(tmpBuffer.asReadOnlyBuffer());
+    final ByteBuffer readOnlyBuffer = data.asReadOnlyBuffer();
+    readOnlyBuffer.position(sizePosition + Integer.BYTES);
+    readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes);
+    return bitmapFactory.mapImmutableBitmap(readOnlyBuffer);
   }
 
   @Override

From 21dcf804eb4ba2f3ed8a3fc176f5241df63f830a Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Tue, 27 Aug 2024 13:38:30 -0700
Subject: [PATCH 61/99] Web console: add ability to issue auxiliary queries to
 speed up data views (#16952)

* Add ability to issue auxiliary queries

* readonly supervisor

* return

* update snapshot

* fix classes
---
 .../form-group-with-info.scss                 |   4 +-
 .../formatted-input/formatted-input.scss      |   2 +-
 .../record-table-pane/record-table-pane.scss  |   2 +-
 .../table-filterable-cell.scss                |   2 +-
 web-console/src/utils/general.tsx             |   8 +
 web-console/src/utils/index.tsx               |   2 -
 web-console/src/utils/query-manager/index.ts  |  22 ++
 .../intermediate-query-state.ts               |   0
 .../query-manager.ts}                         |  60 +++-
 .../utils/{ => query-manager}/query-state.ts  |  16 +-
 .../result-with-auxiliary-work.ts             |  30 ++
 .../datasources-view/datasources-view.tsx     | 334 ++++++++++--------
 .../generic-output-table.scss                 |   2 +-
 .../explore-view/tile-picker/tile-picker.scss |   2 +-
 .../preview-table/preview-table.scss          |   2 +-
 .../supervisors-view.spec.tsx.snap            |   8 +-
 .../supervisors-view/supervisors-view.tsx     | 127 ++++---
 .../column-tree/column-tree.scss              |   2 +-
 .../result-table-pane/result-table-pane.scss  |   2 +-
 19 files changed, 387 insertions(+), 240 deletions(-)
 create mode 100644 web-console/src/utils/query-manager/index.ts
 rename web-console/src/utils/{ => query-manager}/intermediate-query-state.ts (100%)
 rename web-console/src/utils/{query-manager.tsx => query-manager/query-manager.ts} (81%)
 rename web-console/src/utils/{ => query-manager}/query-state.ts (88%)
 create mode 100644 web-console/src/utils/query-manager/result-with-auxiliary-work.ts

diff --git a/web-console/src/components/form-group-with-info/form-group-with-info.scss b/web-console/src/components/form-group-with-info/form-group-with-info.scss
index 237225bf47c..15865eb2fb0 100644
--- a/web-console/src/components/form-group-with-info/form-group-with-info.scss
+++ b/web-console/src/components/form-group-with-info/form-group-with-info.scss
@@ -33,7 +33,7 @@
     }
   }
 
-  .#{$bp-ns}-text-muted .#{$bp-ns}-popover2-target {
+  .#{$bp-ns}-text-muted .#{$bp-ns}-popover-target {
     margin-top: 0;
   }
 
@@ -48,7 +48,7 @@
   }
 }
 
-.#{$bp-ns}-popover2-content {
+.#{$bp-ns}-popover-content {
   .code-block {
     white-space: pre;
     overflow: auto;
diff --git a/web-console/src/components/formatted-input/formatted-input.scss b/web-console/src/components/formatted-input/formatted-input.scss
index 46098e694e2..547620571f6 100644
--- a/web-console/src/components/formatted-input/formatted-input.scss
+++ b/web-console/src/components/formatted-input/formatted-input.scss
@@ -21,7 +21,7 @@
 .formatted-input {
   position: relative;
 
-  & > .#{$bp-ns}-popover2-target {
+  & > .#{$bp-ns}-popover-target {
     position: absolute;
     width: 0;
     right: 0;
diff --git a/web-console/src/components/record-table-pane/record-table-pane.scss b/web-console/src/components/record-table-pane/record-table-pane.scss
index d2d6a86232c..2e5a4cc353c 100644
--- a/web-console/src/components/record-table-pane/record-table-pane.scss
+++ b/web-console/src/components/record-table-pane/record-table-pane.scss
@@ -111,7 +111,7 @@
     width: 100%;
   }
 
-  .#{$bp-ns}-popover2-target {
+  .#{$bp-ns}-popover-target {
     width: 100%;
   }
 }
diff --git a/web-console/src/components/table-filterable-cell/table-filterable-cell.scss b/web-console/src/components/table-filterable-cell/table-filterable-cell.scss
index 95e0d732cf6..6ebf3fee0a6 100644
--- a/web-console/src/components/table-filterable-cell/table-filterable-cell.scss
+++ b/web-console/src/components/table-filterable-cell/table-filterable-cell.scss
@@ -24,7 +24,7 @@
   overflow: hidden;
   text-overflow: ellipsis;
 
-  &.#{$ns}-popover2-target {
+  &.#{$ns}-popover-target {
     display: block; // extra nesting for stronger CSS selectors
   }
 }
diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx
index 7698d3c3af8..765f7e30f64 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -381,6 +381,14 @@ export function findMap<T, Q>(
   return filterMap(xs, f)[0];
 }
 
+export function changeByIndex<T>(
+  xs: readonly T[],
+  i: number,
+  f: (x: T, i: number) => T | undefined,
+): T[] {
+  return filterMap(xs, (x, j) => (j === i ? f(x, i) : x));
+}
+
 export function compact<T>(xs: (T | undefined | false | null | '')[]): T[] {
   return xs.filter(Boolean) as T[];
 }
diff --git a/web-console/src/utils/index.tsx b/web-console/src/utils/index.tsx
index 4daeefe61c1..d9b6d62a66e 100644
--- a/web-console/src/utils/index.tsx
+++ b/web-console/src/utils/index.tsx
@@ -24,14 +24,12 @@ export * from './druid-lookup';
 export * from './druid-query';
 export * from './formatter';
 export * from './general';
-export * from './intermediate-query-state';
 export * from './local-storage-backed-visibility';
 export * from './local-storage-keys';
 export * from './null-mode-detection';
 export * from './object-change';
 export * from './query-action';
 export * from './query-manager';
-export * from './query-state';
 export * from './sanitizers';
 export * from './sql';
 export * from './table-helpers';
diff --git a/web-console/src/utils/query-manager/index.ts b/web-console/src/utils/query-manager/index.ts
new file mode 100644
index 00000000000..cc9d0026145
--- /dev/null
+++ b/web-console/src/utils/query-manager/index.ts
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './intermediate-query-state';
+export * from './query-manager';
+export * from './query-state';
+export * from './result-with-auxiliary-work';
diff --git a/web-console/src/utils/intermediate-query-state.ts b/web-console/src/utils/query-manager/intermediate-query-state.ts
similarity index 100%
rename from web-console/src/utils/intermediate-query-state.ts
rename to web-console/src/utils/query-manager/intermediate-query-state.ts
diff --git a/web-console/src/utils/query-manager.tsx b/web-console/src/utils/query-manager/query-manager.ts
similarity index 81%
rename from web-console/src/utils/query-manager.tsx
rename to web-console/src/utils/query-manager/query-manager.ts
index 9391192475c..5b141c7c89d 100644
--- a/web-console/src/utils/query-manager.tsx
+++ b/web-console/src/utils/query-manager/query-manager.ts
@@ -20,9 +20,11 @@ import type { Canceler, CancelToken } from 'axios';
 import axios from 'axios';
 import debounce from 'lodash.debounce';
 
-import { wait } from './general';
+import { wait } from '../general';
+
 import { IntermediateQueryState } from './intermediate-query-state';
 import { QueryState } from './query-state';
+import { ResultWithAuxiliaryWork } from './result-with-auxiliary-work';
 
 export interface QueryManagerOptions<Q, R, I = never, E extends Error = Error> {
   initState?: QueryState<R, E, I>;
@@ -30,12 +32,12 @@ export interface QueryManagerOptions<Q, R, I = never, E extends Error = Error> {
     query: Q,
     cancelToken: CancelToken,
     setIntermediateQuery: (intermediateQuery: any) => void,
-  ) => Promise<R | IntermediateQueryState<I>>;
+  ) => Promise<R | IntermediateQueryState<I> | ResultWithAuxiliaryWork<R>>;
   backgroundStatusCheck?: (
     state: I,
     query: Q,
     cancelToken: CancelToken,
-  ) => Promise<R | IntermediateQueryState<I>>;
+  ) => Promise<R | IntermediateQueryState<I> | ResultWithAuxiliaryWork<R>>;
   onStateChange?: (queryResolve: QueryState<R, E, I>) => void;
   debounceIdle?: number;
   debounceLoading?: number;
@@ -55,13 +57,13 @@ export class QueryManager<Q, R, I = never, E extends Error = Error> {
     query: Q,
     cancelToken: CancelToken,
     setIntermediateQuery: (intermediateQuery: any) => void,
-  ) => Promise<R | IntermediateQueryState<I>>;
+  ) => Promise<R | IntermediateQueryState<I> | ResultWithAuxiliaryWork<R>>;
 
   private readonly backgroundStatusCheck?: (
     state: I,
     query: Q,
     cancelToken: CancelToken,
-  ) => Promise<R | IntermediateQueryState<I>>;
+  ) => Promise<R | IntermediateQueryState<I> | ResultWithAuxiliaryWork<R>>;
 
   private readonly onStateChange?: (queryResolve: QueryState<R, E, I>) => void;
   private readonly backgroundStatusCheckInitDelay: number;
@@ -120,7 +122,7 @@ export class QueryManager<Q, R, I = never, E extends Error = Error> {
     });
 
     const query = this.lastQuery;
-    let data: R | IntermediateQueryState<I>;
+    let data: R | IntermediateQueryState<I> | ResultWithAuxiliaryWork<R>;
     try {
       data = await this.processQuery(query, cancelToken, (intermediateQuery: any) => {
         this.lastIntermediateQuery = intermediateQuery;
@@ -147,6 +149,7 @@ export class QueryManager<Q, R, I = never, E extends Error = Error> {
           );
         }
         cancelToken.throwIfRequested();
+        if (this.currentQueryId !== myQueryId) return;
 
         this.setState(
           new QueryState<R, E, I>({
@@ -166,6 +169,7 @@ export class QueryManager<Q, R, I = never, E extends Error = Error> {
         if (delay) {
           await wait(delay);
           cancelToken.throwIfRequested();
+          if (this.currentQueryId !== myQueryId) return;
         }
 
         data = await this.backgroundStatusCheck(data.state, query, cancelToken);
@@ -189,12 +193,54 @@ export class QueryManager<Q, R, I = never, E extends Error = Error> {
       backgroundChecks++;
     }
 
+    if (this.currentQueryId !== myQueryId) return;
+
+    if (data instanceof ResultWithAuxiliaryWork && !data.auxiliaryQueries.length) {
+      data = data.result;
+    }
+
+    const lastData = this.state.getSomeData();
+    if (data instanceof ResultWithAuxiliaryWork) {
+      const auxiliaryQueries = data.auxiliaryQueries;
+      const numAuxiliaryQueries = auxiliaryQueries.length;
+      data = data.result;
+
+      this.setState(
+        new QueryState<R, E>({
+          data,
+          auxiliaryLoading: true,
+          lastData,
+        }),
+      );
+
+      try {
+        for (let i = 0; i < numAuxiliaryQueries; i++) {
+          cancelToken.throwIfRequested();
+          if (this.currentQueryId !== myQueryId) return;
+
+          data = await auxiliaryQueries[i](data, cancelToken);
+
+          if (this.currentQueryId !== myQueryId) return;
+          if (i < numAuxiliaryQueries - 1) {
+            // Update data in intermediate state
+            this.setState(
+              new QueryState<R, E>({
+                data,
+                auxiliaryLoading: true,
+                lastData,
+              }),
+            );
+          }
+        }
+      } catch {}
+    }
+
     if (this.currentQueryId !== myQueryId) return;
     this.currentRunCancelFn = undefined;
     this.setState(
       new QueryState<R, E>({
         data,
-        lastData: this.state.getSomeData(),
+        lastData,
       }),
     );
   }
diff --git a/web-console/src/utils/query-state.ts b/web-console/src/utils/query-manager/query-state.ts
similarity index 88%
rename from web-console/src/utils/query-state.ts
rename to web-console/src/utils/query-manager/query-state.ts
index b4387bb9111..807646dc583 100644
--- a/web-console/src/utils/query-state.ts
+++ b/web-console/src/utils/query-manager/query-state.ts
@@ -20,6 +20,7 @@ export type QueryStateState = 'init' | 'loading' | 'data' | 'error';
 
 export interface QueryStateOptions<T, E extends Error = Error, I = never> {
   loading?: boolean;
+  auxiliaryLoading?: boolean;
   intermediate?: I;
   intermediateError?: Error;
   error?: E;
@@ -37,20 +38,19 @@ export class QueryState<T, E extends Error = Error, I = never> {
   public error?: E;
   public data?: T;
   public lastData?: T;
+  public auxiliaryLoading?: boolean;
 
   constructor(opts: QueryStateOptions<T, E, I>) {
     const hasData = typeof opts.data !== 'undefined';
     if (typeof opts.error !== 'undefined') {
-      if (hasData) {
-        throw new Error('can not have both error and data');
-      } else {
-        this.state = 'error';
-        this.error = opts.error;
-      }
+      if (hasData) throw new Error('can not have both error and data');
+      this.state = 'error';
+      this.error = opts.error;
     } else {
       if (hasData) {
         this.state = 'data';
         this.data = opts.data;
+        this.auxiliaryLoading = opts.auxiliaryLoading;
       } else if (opts.loading) {
         this.state = 'loading';
         this.intermediate = opts.intermediate;
@@ -92,4 +92,8 @@ export class QueryState<T, E extends Error = Error, I = never> {
   getSomeData(): T | undefined {
     return this.data || this.lastData;
   }
+
+  isAuxiliaryLoading(): boolean {
+    return Boolean(this.auxiliaryLoading);
+  }
 }
diff --git a/web-console/src/utils/query-manager/result-with-auxiliary-work.ts b/web-console/src/utils/query-manager/result-with-auxiliary-work.ts
new file mode 100644
index 00000000000..fae424f3b10
--- /dev/null
+++ b/web-console/src/utils/query-manager/result-with-auxiliary-work.ts
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { CancelToken } from 'axios';
+
+export type AuxiliaryQueryFn<R> = (result: R, cancelToken: CancelToken) => Promise<R>;
+export class ResultWithAuxiliaryWork<R> {
+  public readonly result: R;
+  public readonly auxiliaryQueries: AuxiliaryQueryFn<R>[];
+
+  constructor(result: R, auxiliaryQueries: AuxiliaryQueryFn<R>[]) {
+    this.result = result;
+    this.auxiliaryQueries = auxiliaryQueries;
+  }
+}
diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx
index da5f8dbbff1..69c6302c654 100644
--- a/web-console/src/views/datasources-view/datasources-view.tsx
+++ b/web-console/src/views/datasources-view/datasources-view.tsx
@@ -55,7 +55,7 @@ import { formatCompactionInfo, zeroCompactionStatus } from '../../druid-models';
 import type { Capabilities, CapabilitiesMode } from '../../helpers';
 import { STANDARD_TABLE_PAGE_SIZE, STANDARD_TABLE_PAGE_SIZE_OPTIONS } from '../../react-table';
 import { Api, AppToaster } from '../../singletons';
-import type { NumberLike } from '../../utils';
+import type { AuxiliaryQueryFn, NumberLike } from '../../utils';
 import {
   assemble,
   compact,
@@ -77,6 +77,7 @@ import {
   queryDruidSql,
   QueryManager,
   QueryState,
+  ResultWithAuxiliaryWork,
   twoLines,
 } from '../../utils';
 import type { BasicAction } from '../../utils/basic-action';
@@ -262,8 +263,7 @@ function countRunningTasks(runningTasks: Record<string, number> | undefined): nu
   return sum(Object.values(runningTasks));
 }
 
-function formatRunningTasks(runningTasks: Record<string, number> | undefined): string {
-  if (!runningTasks) return 'n/a';
+function formatRunningTasks(runningTasks: Record<string, number>): string {
   const runningTaskEntries = Object.entries(runningTasks);
   if (!runningTaskEntries.length) return 'No running tasks';
   return moveToEnd(
@@ -472,141 +472,180 @@ GROUP BY 1, 2`;
           throw new Error(`must have SQL or coordinator access`);
         }
 
-        let runningTasksByDatasource: Record<string, Record<string, number>> = {};
-        if (visibleColumns.shown('Running tasks')) {
-          try {
-            if (capabilities.hasSql()) {
-              const runningTasks = await queryDruidSql<RunningTaskRow>({
-                query: DatasourcesView.RUNNING_TASK_SQL,
-              });
+        const auxiliaryQueries: AuxiliaryQueryFn<DatasourcesAndDefaultRules>[] = [];
 
-              runningTasksByDatasource = groupByAsMap(
-                runningTasks,
-                x => x.datasource,
-                xs =>
-                  groupByAsMap(
-                    xs,
-                    x => normalizeTaskType(x.type),
-                    ys => sum(ys, y => y.num_running_tasks),
-                  ),
-              );
-            } else if (capabilities.hasOverlordAccess()) {
-              const taskList = (await Api.instance.get(`/druid/indexer/v1/tasks?state=running`))
-                .data;
-              runningTasksByDatasource = groupByAsMap(
-                taskList,
-                (t: any) => t.dataSource,
-                xs =>
-                  groupByAsMap(
-                    xs,
-                    x => normalizeTaskType(x.type),
-                    ys => ys.length,
-                  ),
-              );
-            } else {
-              throw new Error(`must have SQL or overlord access`);
-            }
-          } catch (e) {
-            AppToaster.show({
-              icon: IconNames.ERROR,
-              intent: Intent.DANGER,
-              message: 'Could not get running task counts',
+        if (visibleColumns.shown('Running tasks')) {
+          if (capabilities.hasSql()) {
+            auxiliaryQueries.push(async (datasourcesAndDefaultRules, cancelToken) => {
+              try {
+                const runningTasks = await queryDruidSql<RunningTaskRow>(
+                  {
+                    query: DatasourcesView.RUNNING_TASK_SQL,
+                  },
+                  cancelToken,
+                );
+
+                const runningTasksByDatasource = groupByAsMap(
+                  runningTasks,
+                  x => x.datasource,
+                  xs =>
+                    groupByAsMap(
+                      xs,
+                      x => normalizeTaskType(x.type),
+                      ys => sum(ys, y => y.num_running_tasks),
+                    ),
+                );
+
+                return {
+                  ...datasourcesAndDefaultRules,
+                  datasources: datasourcesAndDefaultRules.datasources.map(ds => ({
+                    ...ds,
+                    runningTasks: runningTasksByDatasource[ds.datasource] || {},
+                  })),
+                };
+              } catch {
+                AppToaster.show({
+                  icon: IconNames.ERROR,
+                  intent: Intent.DANGER,
+                  message: 'Could not get running task counts',
+                });
+                return datasourcesAndDefaultRules;
+              }
+            });
+          }
+
+          if (capabilities.hasOverlordAccess()) {
+            auxiliaryQueries.push(async (datasourcesAndDefaultRules, cancelToken) => {
+              try {
+                const runningTasks = await queryDruidSql<RunningTaskRow>(
+                  {
+                    query: DatasourcesView.RUNNING_TASK_SQL,
+                  },
+                  cancelToken,
+                );
+
+                const runningTasksByDatasource = groupByAsMap(
+                  runningTasks,
+                  x => x.datasource,
+                  xs =>
+                    groupByAsMap(
+                      xs,
+                      x => normalizeTaskType(x.type),
+                      ys => sum(ys, y => y.num_running_tasks),
+                    ),
+                );
+
+                return {
+                  ...datasourcesAndDefaultRules,
+                  datasources: datasourcesAndDefaultRules.datasources.map(ds => ({
+                    ...ds,
+                    runningTasks: runningTasksByDatasource[ds.datasource] || {},
+                  })),
+                };
+              } catch {
+                AppToaster.show({
+                  icon: IconNames.ERROR,
+                  intent: Intent.DANGER,
+                  message: 'Could not get running task counts',
+                });
+                return datasourcesAndDefaultRules;
+              }
             });
           }
         }
 
-        if (!capabilities.hasCoordinatorAccess()) {
-          return {
-            datasources: datasources.map(ds => ({ ...ds, rules: [] })),
-            defaultRules: [],
-          };
-        }
-
-        const seen = countBy(datasources, x => x.datasource);
-
         let unused: string[] = [];
-        if (showUnused) {
-          try {
-            unused = (
-              await Api.instance.get<string[]>(
-                '/druid/coordinator/v1/metadata/datasources?includeUnused',
-              )
-            ).data.filter(d => !seen[d]);
-          } catch {
-            AppToaster.show({
-              icon: IconNames.ERROR,
-              intent: Intent.DANGER,
-              message: 'Could not get the list of unused datasources',
-            });
+        if (capabilities.hasCoordinatorAccess()) {
+          // Unused
+          const seen = countBy(datasources, x => x.datasource);
+          if (showUnused) {
+            try {
+              unused = (
+                await Api.instance.get<string[]>(
+                  '/druid/coordinator/v1/metadata/datasources?includeUnused',
+                )
+              ).data.filter(d => !seen[d]);
+            } catch {
+              AppToaster.show({
+                icon: IconNames.ERROR,
+                intent: Intent.DANGER,
+                message: 'Could not get the list of unused datasources',
+              });
+            }
           }
-        }
 
-        let rules: Record<string, Rule[]> = {};
-        try {
-          rules = (await Api.instance.get<Record<string, Rule[]>>('/druid/coordinator/v1/rules'))
-            .data;
-        } catch {
-          AppToaster.show({
-            icon: IconNames.ERROR,
-            intent: Intent.DANGER,
-            message: 'Could not get load rules',
+          // Rules
+          auxiliaryQueries.push(async (datasourcesAndDefaultRules, cancelToken) => {
+            try {
+              const rules: Record<string, Rule[]> = (
+                await Api.instance.get<Record<string, Rule[]>>('/druid/coordinator/v1/rules', {
+                  cancelToken,
+                })
+              ).data;
+
+              return {
+                datasources: datasourcesAndDefaultRules.datasources.map(ds => ({
+                  ...ds,
+                  rules: rules[ds.datasource] || [],
+                })),
+                defaultRules: rules[DEFAULT_RULES_KEY],
+              };
+            } catch {
+              AppToaster.show({
+                icon: IconNames.ERROR,
+                intent: Intent.DANGER,
+                message: 'Could not get load rules',
+              });
+              return datasourcesAndDefaultRules;
+            }
+          });
+
+          // Compaction
+          auxiliaryQueries.push(async (datasourcesAndDefaultRules, cancelToken) => {
+            try {
+              const compactionConfigsResp = await Api.instance.get<{
+                compactionConfigs: CompactionConfig[];
+              }>('/druid/coordinator/v1/config/compaction', { cancelToken });
+              const compactionConfigs = lookupBy(
+                compactionConfigsResp.data.compactionConfigs || [],
+                c => c.dataSource,
+              );
+
+              const compactionStatusesResp = await Api.instance.get<{
+                latestStatus: CompactionStatus[];
+              }>('/druid/coordinator/v1/compaction/status', { cancelToken });
+              const compactionStatuses = lookupBy(
+                compactionStatusesResp.data.latestStatus || [],
+                c => c.dataSource,
+              );
+
+              return {
+                ...datasourcesAndDefaultRules,
+                datasources: datasourcesAndDefaultRules.datasources.map(ds => ({
+                  ...ds,
+                  compaction: {
+                    config: compactionConfigs[ds.datasource],
+                    status: compactionStatuses[ds.datasource],
+                  },
+                })),
+              };
+            } catch {
+              AppToaster.show({
+                icon: IconNames.ERROR,
+                intent: Intent.DANGER,
+                message: 'Could not get compaction information',
+              });
+              return datasourcesAndDefaultRules;
+            }
           });
         }
 
-        let compactionConfigs: Record<string, CompactionConfig> | undefined;
-        try {
-          const compactionConfigsResp = await Api.instance.get<{
-            compactionConfigs: CompactionConfig[];
-          }>('/druid/coordinator/v1/config/compaction');
-          compactionConfigs = lookupBy(
-            compactionConfigsResp.data.compactionConfigs || [],
-            c => c.dataSource,
-          );
-        } catch {
-          AppToaster.show({
-            icon: IconNames.ERROR,
-            intent: Intent.DANGER,
-            message: 'Could not get compaction configs',
-          });
-        }
-
-        let compactionStatuses: Record<string, CompactionStatus> | undefined;
-        if (compactionConfigs) {
-          // Don't bother getting the statuses if we can not even get the configs
-          try {
-            const compactionStatusesResp = await Api.instance.get<{
-              latestStatus: CompactionStatus[];
-            }>('/druid/coordinator/v1/compaction/status');
-            compactionStatuses = lookupBy(
-              compactionStatusesResp.data.latestStatus || [],
-              c => c.dataSource,
-            );
-          } catch {
-            AppToaster.show({
-              icon: IconNames.ERROR,
-              intent: Intent.DANGER,
-              message: 'Could not get compaction statuses',
-            });
-          }
-        }
-
-        return {
-          datasources: datasources.concat(unused.map(makeUnusedDatasource)).map(ds => {
-            return {
-              ...ds,
-              runningTasks: runningTasksByDatasource[ds.datasource] || {},
-              rules: rules[ds.datasource],
-              compaction:
-                compactionConfigs && compactionStatuses
-                  ? {
-                      config: compactionConfigs[ds.datasource],
-                      status: compactionStatuses[ds.datasource],
-                    }
-                  : undefined,
-            };
-          }),
-          defaultRules: rules[DEFAULT_RULES_KEY],
-        };
+        return new ResultWithAuxiliaryWork(
+          {
+            datasources: datasources.concat(unused.map(makeUnusedDatasource)),
+          },
+          auxiliaryQueries,
+        );
       },
       onStateChange: datasourcesAndDefaultRulesState => {
         this.setState({
@@ -1271,15 +1310,19 @@ GROUP BY 1, 2`;
             accessor: d => countRunningTasks(d.runningTasks),
             filterable: false,
             width: 200,
-            Cell: ({ original }) => (
-              <TableClickableCell
-                onClick={() => goToTasks(original.datasource)}
-                hoverIcon={IconNames.ARROW_TOP_RIGHT}
-                title="Go to tasks"
-              >
-                {formatRunningTasks(original.runningTasks)}
-              </TableClickableCell>
-            ),
+            Cell: ({ original }) => {
+              const { runningTasks } = original;
+              if (!runningTasks) return;
+              return (
+                <TableClickableCell
+                  onClick={() => goToTasks(original.datasource)}
+                  hoverIcon={IconNames.ARROW_TOP_RIGHT}
+                  title="Go to tasks"
+                >
+                  {formatRunningTasks(runningTasks)}
+                </TableClickableCell>
+              );
+            },
           },
           {
             Header: twoLines('Segment rows', 'minimum / average / maximum'),
@@ -1451,6 +1494,7 @@ GROUP BY 1, 2`;
             width: 180,
             Cell: ({ original }) => {
               const { datasource, compaction } = original as Datasource;
+              if (!compaction) return;
               return (
                 <TableClickableCell
                   disabled={!compaction}
@@ -1465,7 +1509,7 @@ GROUP BY 1, 2`;
                   }}
                   hoverIcon={IconNames.EDIT}
                 >
-                  {compaction ? formatCompactionInfo(compaction) : 'Could not get compaction info'}
+                  {formatCompactionInfo(compaction)}
                 </TableClickableCell>
               );
             },
@@ -1485,9 +1529,7 @@ GROUP BY 1, 2`;
             className: 'padded',
             Cell: ({ original }) => {
               const { compaction } = original as Datasource;
-              if (!compaction) {
-                return 'Could not get compaction info';
-              }
+              if (!compaction) return;
 
               const { status } = compaction;
               if (!status || zeroCompactionStatus(status)) {
@@ -1543,9 +1585,7 @@ GROUP BY 1, 2`;
             className: 'padded',
             Cell: ({ original }) => {
               const { compaction } = original as Datasource;
-              if (!compaction) {
-                return 'Could not get compaction info';
-              }
+              if (!compaction) return;
 
               const { status } = compaction;
               if (!status) {
@@ -1569,6 +1609,8 @@ GROUP BY 1, 2`;
             width: 200,
             Cell: ({ original }) => {
               const { datasource, rules } = original as Datasource;
+              if (!rules) return;
+
               return (
                 <TableClickableCell
                   disabled={!defaultRules}
@@ -1577,17 +1619,17 @@ GROUP BY 1, 2`;
                     this.setState({
                       retentionDialogOpenOn: {
                         datasource,
-                        rules: rules || [],
+                        rules,
                       },
                     });
                   }}
                   hoverIcon={IconNames.EDIT}
                 >
-                  {rules?.length
+                  {rules.length
                     ? DatasourcesView.formatRules(rules)
                     : defaultRules
                     ? `Cluster default: ${DatasourcesView.formatRules(defaultRules)}`
-                    : 'Could not get default rules'}
+                    : ''}
                 </TableClickableCell>
               );
             },
diff --git a/web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.scss b/web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.scss
index ea9f6a24a9b..31733db7828 100644
--- a/web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.scss
+++ b/web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.scss
@@ -111,7 +111,7 @@
     width: 100%;
   }
 
-  .#{$bp-ns}-popover2-target {
+  .#{$bp-ns}-popover-target {
     width: 100%;
   }
 
diff --git a/web-console/src/views/explore-view/tile-picker/tile-picker.scss b/web-console/src/views/explore-view/tile-picker/tile-picker.scss
index 95d0dbcfb46..679965670b3 100644
--- a/web-console/src/views/explore-view/tile-picker/tile-picker.scss
+++ b/web-console/src/views/explore-view/tile-picker/tile-picker.scss
@@ -23,7 +23,7 @@
     flex: 1 1 auto;
   }
 
-  .more-button.#{$ns}-popover2-target {
+  .more-button.#{$ns}-popover-target {
     flex: 0;
   }
 }
diff --git a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.scss b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.scss
index 462a852e8df..a18bc291fd7 100644
--- a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.scss
+++ b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.scss
@@ -152,7 +152,7 @@
     width: 100%;
   }
 
-  .#{$bp-ns}-popover2-target {
+  .#{$bp-ns}-popover-target {
     width: 100%;
   }
 
diff --git a/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap b/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
index 4b531c0bc44..e3458f4749c 100644
--- a/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
+++ b/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
@@ -82,14 +82,14 @@ exports[`SupervisorsView matches snapshot 1`] = `
             "label": "status API",
             "text": "Aggregate lag",
           },
-          {
-            "label": "status API",
-            "text": "Recent errors",
-          },
           {
             "label": "stats API",
             "text": "Stats",
           },
+          {
+            "label": "status API",
+            "text": "Recent errors",
+          },
         ]
       }
       onChange={[Function]}
diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx
index 648186d45bc..41d8ad2cd0f 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.tsx
+++ b/web-console/src/views/supervisors-view/supervisors-view.tsx
@@ -61,9 +61,10 @@ import {
   sqlQueryCustomTableFilter,
 } from '../../react-table';
 import { Api, AppToaster } from '../../singletons';
-import type { TableState } from '../../utils';
+import type { AuxiliaryQueryFn, TableState } from '../../utils';
 import {
   assemble,
+  changeByIndex,
   checkedCircleIcon,
   deepGet,
   filterMap,
@@ -73,6 +74,7 @@ import {
   formatRate,
   getDruidErrorMessage,
   hasPopoverOpen,
+  isNumberLike,
   LocalStorageBackedVisibility,
   LocalStorageKeys,
   nonEmptyArray,
@@ -81,6 +83,7 @@ import {
   queryDruidSql,
   QueryManager,
   QueryState,
+  ResultWithAuxiliaryWork,
   sortedToOrderByClause,
   twoLines,
 } from '../../utils';
@@ -96,8 +99,8 @@ const SUPERVISOR_TABLE_COLUMNS: TableColumnSelectorColumn[] = [
   'Configured tasks',
   { text: 'Running tasks', label: 'status API' },
   { text: 'Aggregate lag', label: 'status API' },
-  { text: 'Recent errors', label: 'status API' },
   { text: 'Stats', label: 'stats API' },
+  { text: 'Recent errors', label: 'status API' },
 ];
 
 const ROW_STATS_KEYS: RowStatsKey[] = ['1m', '5m', '15m'];
@@ -118,14 +121,14 @@ interface SupervisorQuery extends TableState {
 }
 
 interface SupervisorQueryResultRow {
-  supervisor_id: string;
-  type: string;
-  source: string;
-  detailed_state: string;
-  spec?: IngestionSpec;
-  suspended: boolean;
-  status?: SupervisorStatus;
-  stats?: any;
+  readonly supervisor_id: string;
+  readonly type: string;
+  readonly source: string;
+  readonly detailed_state: string;
+  readonly spec?: IngestionSpec;
+  readonly suspended: boolean;
+  readonly status?: SupervisorStatus;
+  readonly stats?: any;
 }
 
 export interface SupervisorsViewProps {
@@ -253,19 +256,18 @@ export class SupervisorsView extends React.PureComponent<
             page ? `OFFSET ${page * pageSize}` : undefined,
           ).join('\n');
           setIntermediateQuery(sqlQuery);
-          supervisors = await queryDruidSql<SupervisorQueryResultRow>(
-            {
-              query: sqlQuery,
-            },
-            cancelToken,
-          );
-
-          for (const supervisor of supervisors) {
+          supervisors = (
+            await queryDruidSql<SupervisorQueryResultRow>(
+              {
+                query: sqlQuery,
+              },
+              cancelToken,
+            )
+          ).map(supervisor => {
             const spec: any = supervisor.spec;
-            if (typeof spec === 'string') {
-              supervisor.spec = JSONBig.parse(spec);
-            }
-          }
+            if (typeof spec !== 'string') return supervisor;
+            return { ...supervisor, spec: JSONBig.parse(spec) };
+          });
         } else if (capabilities.hasOverlordAccess()) {
           const supervisorList = (
             await Api.instance.get('/druid/indexer/v1/supervisor?full', { cancelToken })
@@ -302,54 +304,48 @@ export class SupervisorsView extends React.PureComponent<
           throw new Error(`must have SQL or overlord access`);
         }
 
+        const auxiliaryQueries: AuxiliaryQueryFn<SupervisorQueryResultRow[]>[] = [];
         if (capabilities.hasOverlordAccess()) {
-          let showIssue = (message: string) => {
-            showIssue = () => {}; // Only show once
-            AppToaster.show({
-              icon: IconNames.ERROR,
-              intent: Intent.DANGER,
-              message,
-            });
-          };
-
           if (visibleColumns.shown('Running tasks', 'Aggregate lag', 'Recent errors')) {
-            try {
-              for (const supervisor of supervisors) {
-                cancelToken.throwIfRequested();
-                supervisor.status = (
-                  await Api.instance.get(
-                    `/druid/indexer/v1/supervisor/${Api.encodePath(
-                      supervisor.supervisor_id,
-                    )}/status`,
-                    { cancelToken, timeout: STATUS_API_TIMEOUT },
-                  )
-                ).data;
-              }
-            } catch (e) {
-              showIssue('Could not get status');
-            }
+            auxiliaryQueries.push(
+              ...supervisors.map(
+                (supervisor, i): AuxiliaryQueryFn<SupervisorQueryResultRow[]> =>
+                  async (rows, cancelToken) => {
+                    const status = (
+                      await Api.instance.get(
+                        `/druid/indexer/v1/supervisor/${Api.encodePath(
+                          supervisor.supervisor_id,
+                        )}/status`,
+                        { cancelToken, timeout: STATUS_API_TIMEOUT },
+                      )
+                    ).data;
+                    return changeByIndex(rows, i, row => ({ ...row, status }));
+                  },
+              ),
+            );
           }
 
           if (visibleColumns.shown('Stats')) {
-            try {
-              for (const supervisor of supervisors) {
-                cancelToken.throwIfRequested();
-                supervisor.stats = (
-                  await Api.instance.get(
-                    `/druid/indexer/v1/supervisor/${Api.encodePath(
-                      supervisor.supervisor_id,
-                    )}/stats`,
-                    { cancelToken, timeout: STATS_API_TIMEOUT },
-                  )
-                ).data;
-              }
-            } catch (e) {
-              showIssue('Could not get stats');
-            }
+            auxiliaryQueries.push(
+              ...supervisors.map(
+                (supervisor, i): AuxiliaryQueryFn<SupervisorQueryResultRow[]> =>
+                  async (rows, cancelToken) => {
+                    const stats = (
+                      await Api.instance.get(
+                        `/druid/indexer/v1/supervisor/${Api.encodePath(
+                          supervisor.supervisor_id,
+                        )}/stats`,
+                        { cancelToken, timeout: STATS_API_TIMEOUT },
+                      )
+                    ).data;
+                    return changeByIndex(rows, i, row => ({ ...row, stats }));
+                  },
+              ),
+            );
           }
         }
 
-        return supervisors;
+        return new ResultWithAuxiliaryWork(supervisors, auxiliaryQueries);
       },
       onStateChange: supervisorsState => {
         this.setState({
@@ -790,7 +786,7 @@ export class SupervisorsView extends React.PureComponent<
                   );
                 }
               } else {
-                label = 'n/a';
+                label = '';
               }
               return (
                 <TableClickableCell
@@ -812,7 +808,7 @@ export class SupervisorsView extends React.PureComponent<
             sortable: false,
             className: 'padded',
             show: visibleColumns.shown('Aggregate lag'),
-            Cell: ({ value }) => formatInteger(value),
+            Cell: ({ value }) => (isNumberLike(value) ? formatInteger(value) : null),
           },
           {
             Header: twoLines(
@@ -899,13 +895,14 @@ export class SupervisorsView extends React.PureComponent<
             sortable: false,
             show: visibleColumns.shown('Recent errors'),
             Cell: ({ value, original }) => {
+              if (!value) return null;
               return (
                 <TableClickableCell
                   onClick={() => this.onSupervisorDetail(original)}
                   hoverIcon={IconNames.SEARCH_TEMPLATE}
                   title="See errors"
                 >
-                  {pluralIfNeeded(value?.length, 'error')}
+                  {pluralIfNeeded(value.length, 'error')}
                 </TableClickableCell>
               );
             },
diff --git a/web-console/src/views/workbench-view/column-tree/column-tree.scss b/web-console/src/views/workbench-view/column-tree/column-tree.scss
index 2371c145b59..d13431f0891 100644
--- a/web-console/src/views/workbench-view/column-tree/column-tree.scss
+++ b/web-console/src/views/workbench-view/column-tree/column-tree.scss
@@ -59,7 +59,7 @@
       animation: druid-glow 1s infinite alternate;
     }
 
-    .#{$bp-ns}-popover2-target {
+    .#{$bp-ns}-popover-target {
       width: 188px;
       display: inline-block;
       cursor: pointer;
diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.scss b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.scss
index 366deb7956f..6ab4629d3ef 100644
--- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.scss
+++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.scss
@@ -111,7 +111,7 @@
     width: 100%;
   }
 
-  .#{$bp-ns}-popover2-target {
+  .#{$bp-ns}-popover-target {
     width: 100%;
   }
 

From 3e031b9dc22256dcd92fbcf3ad242dca41e264f8 Mon Sep 17 00:00:00 2001
From: Jill Osborne <jill.osborne@imply.io>
Date: Tue, 27 Aug 2024 22:27:13 +0100
Subject: [PATCH 62/99] Add dynamic query params example (#16964)

Co-authored-by: Victoria Lim <vtlim@users.noreply.github.com>
---
 docs/querying/sql.md | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/querying/sql.md b/docs/querying/sql.md
index c0241bef879..6a47ac3a4c0 100644
--- a/docs/querying/sql.md
+++ b/docs/querying/sql.md
@@ -398,6 +398,21 @@ at execution time. To use dynamic parameters, replace any literal in the query w
 corresponding parameter value when you execute the query. Parameters are bound to the placeholders in the order in
 which they are passed. Parameters are supported in both the [HTTP POST](../api-reference/sql-api.md) and [JDBC](../api-reference/sql-jdbc.md) APIs.
 
+Druid supports double and null values in arrays for dynamic queries.
+The following example query uses the [ARRAY_CONTAINS](./sql-functions.md#array_contains) function to return `doubleArrayColumn` when the reference array `[-25.7, null, 36.85]` contains all elements of the value of `doubleArrayColumn`:
+
+```sql
+{
+   "query": "SELECT doubleArrayColumn from druid.table where ARRAY_CONTAINS(?, doubleArrayColumn)",
+   "parameters": [
+      {
+        "type": "ARRAY",
+        "value": [-25.7, null, 36.85]
+      }
+   ]
+}
+```
+
 In certain cases, using dynamic parameters in expressions can cause type inference issues which cause your query to fail, for example:
 
 ```sql
@@ -435,4 +450,4 @@ To use the reserved keywords in queries, enclose them in double quotation marks.
 
 ```sql
 SELECT "PARTITIONED" from druid.table
-```
\ No newline at end of file
+```

From e562dd3ac6d4974d17c1816da7e5f4d6c7ecb309 Mon Sep 17 00:00:00 2001
From: Charles Smith <techdocsmith@gmail.com>
Date: Tue, 27 Aug 2024 14:27:23 -0700
Subject: [PATCH 63/99] Docs: note on iceberg (#16955)

Co-authored-by: Victoria Lim <vtlim@users.noreply.github.com>
---
 docs/development/extensions-contrib/iceberg.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/development/extensions-contrib/iceberg.md b/docs/development/extensions-contrib/iceberg.md
index f4a1735ed3b..434e21c656c 100644
--- a/docs/development/extensions-contrib/iceberg.md
+++ b/docs/development/extensions-contrib/iceberg.md
@@ -22,6 +22,9 @@ title: "Iceberg extension"
   ~ under the License.
   -->
 
+<!-- If the URL changes for this topic, make modifications
+to Apache Iceberg docs: https://github.com/apache/iceberg/blob/main/docs/mkdocs.yml -->
+
 ## Iceberg Ingest extension
 
 Apache Iceberg is an open table format for huge analytic datasets. [IcebergInputSource](../../ingestion/input-sources.md#iceberg-input-source) lets you ingest data stored in the Iceberg table format into Apache Druid. To use the iceberg extension, add the `druid-iceberg-extensions` to the list of loaded extensions. See [Loading extensions](../../configuration/extensions.md#loading-extensions) for more information.

From 862ccda59b711557376aab66abf8280cadb25e02 Mon Sep 17 00:00:00 2001
From: Virushade <70288012+GWphua@users.noreply.github.com>
Date: Wed, 28 Aug 2024 11:28:43 +0800
Subject: [PATCH 64/99] Disable automatic search refresh every for keystroke
 (#16963)

* Disable refreshing page everytime keystroke is detected

* Lengthen input debounce time to 1s

* Run Prettier to pass stylecheck
---
 .../src/react-table/react-table-inputs.tsx    | 29 +++++++++++++++----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/web-console/src/react-table/react-table-inputs.tsx b/web-console/src/react-table/react-table-inputs.tsx
index 06bacd1a06d..2d02cb71b78 100644
--- a/web-console/src/react-table/react-table-inputs.tsx
+++ b/web-console/src/react-table/react-table-inputs.tsx
@@ -19,7 +19,7 @@
 import { Button, HTMLSelect, Icon, InputGroup, Menu, MenuItem, Popover } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
 import classNames from 'classnames';
-import React, { useState } from 'react';
+import React, { useEffect, useState } from 'react';
 import type { Column, ReactTableFunction } from 'react-table';
 
 import {
@@ -39,8 +39,10 @@ interface FilterRendererProps {
 }
 
 export function GenericFilterInput({ column, filter, onChange, key }: FilterRendererProps) {
+  const INPUT_DEBOUNCE_TIME_IN_MILLISECONDS = 1000;
   const [menuOpen, setMenuOpen] = useState(false);
   const [focusedText, setFocusedText] = useState<string | undefined>();
+  const [debouncedValue, setDebouncedValue] = useState<string | undefined>();
 
   const enableComparisons = String(column.headerClassName).includes('enable-comparisons');
 
@@ -49,6 +51,19 @@ export function GenericFilterInput({ column, filter, onChange, key }: FilterRend
     needle: '',
   };
 
+  useEffect(() => {
+    const handler = setTimeout(() => {
+      if (focusedText !== undefined && focusedText !== debouncedValue) {
+        onChange(combineModeAndNeedle(mode, focusedText));
+        setDebouncedValue(focusedText);
+      }
+    }, INPUT_DEBOUNCE_TIME_IN_MILLISECONDS);
+
+    return () => {
+      clearTimeout(handler);
+    };
+  }, [focusedText, debouncedValue, mode, onChange]);
+
   return (
     <InputGroup
       className={classNames('generic-filter-input', {
@@ -79,15 +94,17 @@ export function GenericFilterInput({ column, filter, onChange, key }: FilterRend
         </Popover>
       }
       value={focusedText ?? needle}
-      onChange={e => {
-        const enteredText = e.target.value;
-        setFocusedText(enteredText);
-        onChange(combineModeAndNeedle(mode, enteredText));
+      onChange={e => setFocusedText(e.target.value)}
+      onKeyDown={e => {
+        if (e.key === 'Enter') {
+          const inputValue = (e.target as HTMLInputElement).value;
+          setDebouncedValue(undefined); // Reset debounce to avoid duplicate triggers
+          onChange(combineModeAndNeedle(mode, inputValue));
+        }
       }}
       rightElement={
         filter ? <Button icon={IconNames.CROSS} minimal onClick={() => onChange('')} /> : undefined
       }
-      onFocus={() => setFocusedText(needle)}
       onBlur={e => {
         setFocusedText(undefined);
         if (filter && !e.target.value) onChange('');

From fbd305af0f1d300f27d2ff8e67cf9937c55c56e2 Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Wed, 28 Aug 2024 11:32:47 +0530
Subject: [PATCH 65/99] MSQ WF: Batch multiple PARTITION BY keys for processing
 (#16823)

Currently, if we have a query with window function having PARTITION BY xyz, and we have a million unique values for xyz each having 1 row, we'd end up creating a million individual RACs for processing, each having a single row. This is unnecessary, and we can batch the PARTITION BY keys together for processing, and process them only when we can't batch further rows to adhere to maxRowsMaterialized config.

The previous iteration of this PR was simplifying WindowOperatorQueryFrameProcessor to run all operators on all the rows instead of creating smaller RACs per partition by key. That approach was discarded in favor of the batching approach, and the details are summarized here: #16823 (comment).
---
 .../WindowOperatorQueryFrameProcessor.java    | 327 ++++++++----------
 .../msq/querykit/FrameProcessorTestBase.java  | 108 ++++++
 ...WindowOperatorQueryFrameProcessorTest.java | 235 +++++++++++++
 .../SortMergeJoinFrameProcessorTest.java      |  69 +---
 .../QueryResultsFrameProcessorTest.java       |  26 +-
 .../scan/ScanQueryFrameProcessorTest.java     |  25 +-
 .../MapOfColumnsRowsAndColumns.java           |   9 +-
 7 files changed, 509 insertions(+), 290 deletions(-)
 create mode 100644 extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/FrameProcessorTestBase.java
 create mode 100644 extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessorTest.java

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
index 6b28c0263a8..3dc62f3a60d 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
@@ -80,7 +80,6 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
   private final WritableFrameChannel outputChannel;
   private final FrameWriterFactory frameWriterFactory;
   private final FrameReader frameReader;
-  private final ArrayList<ResultRow> objectsOfASingleRac;
   private final int maxRowsMaterialized;
   private long currentAllocatorCapacity; // Used for generating FrameRowTooLargeException if needed
   private Cursor frameCursor = null;
@@ -95,6 +94,9 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
   // Type strategies are pushed in the same order as column types in frameReader.signature()
   private final NullableTypeStrategy[] typeStrategies;
 
+  private final ArrayList<ResultRow> rowsToProcess;
+  private int lastPartitionIndex = -1;
+
   public WindowOperatorQueryFrameProcessor(
       WindowOperatorQuery query,
       ReadableFrameChannel inputChannel,
@@ -116,7 +118,7 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
     this.query = query;
     this.frameRowsAndCols = new ArrayList<>();
     this.resultRowAndCols = new ArrayList<>();
-    this.objectsOfASingleRac = new ArrayList<>();
+    this.rowsToProcess = new ArrayList<>();
     this.maxRowsMaterialized = maxRowsMaterializedInWindow;
     this.partitionColumnNames = partitionColumnNames;
 
@@ -153,188 +155,117 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
   public ReturnOrAwait<Object> runIncrementally(IntSet readableInputs)
   {
     /*
-     *
-     * PARTITION BY A ORDER BY B
-     *
-     * Frame 1   -> rac1
-     * A  B
-     * 1, 2
-     * 1, 3
-     * 2, 1 --> key changed
-     * 2, 2
-     *
-     *
-     * Frame 2 -> rac2
-     * 3, 1 --> key changed
-     * 3, 2
-     * 3, 3
-     * 3, 4
-     *
-     * Frame 3 -> rac3
-     *
-     * 3, 5
-     * 3, 6
-     * 4, 1 --> key changed
-     * 4, 2
-     *
-     * In case of empty OVER clause, all these racs need to be added to a single rows and columns
-     * to be processed. The way we can do this is to use a ConcatRowsAndColumns
-     * ConcatRC [rac1, rac2, rac3]
-     * Run all ops on this
-     *
-     *
-     * The flow would look like:
-     * 1. Validate if the operator doesn't have any OVER() clause with PARTITION BY for this stage.
-     * 2. If 1 is true make a giant rows and columns (R&C) using concat as shown above
-     *    Let all operators run amok on that R&C
-     * 3. If 1 is false
-     *    Read a frame
-     *    keep the older row in a class variable
-     *    check row by row and compare current with older row to check if partition boundary is reached
-     *    when frame partition by changes
-     *    create R&C for those particular set of columns, they would have the same partition key
-     *    output will be a single R&C
-     *    write to output channel
-     *
-     *
-     *  Future thoughts: {@link https://github.com/apache/druid/issues/16126}
-     *
-     *  1. We are writing 1 partition to each frame in this way. In case of high cardinality data
-     *      we will be making a large number of small frames. We can have a check to keep size of frame to a value
-     *      say 20k rows and keep on adding to the same pending frame and not create a new frame
-     *
-     *  2. Current approach with R&C and operators materialize a single R&C for processing. In case of data
-     *     with low cardinality a single R&C might be too big to consume. Same for the case of empty OVER() clause
-     *     Most of the window operations like SUM(), RANK(), RANGE() etc. can be made with 2 passes of the data.
-     *     We might think to reimplement them in the MSQ way so that we do not have to materialize so much data
+     There are 2 scenarios:
+
+     *** Scenario 1: Query has atleast one window function with an OVER() clause without a PARTITION BY ***
+
+     In this scenario, we add all the RACs to a single RowsAndColumns to be processed. We do it via ConcatRowsAndColumns, and run all the operators on the ConcatRowsAndColumns.
+     This is done because we anyway need to run the operators on the entire set of rows when we have an OVER() clause without a PARTITION BY.
+     This scenario corresponds to partitionColumnNames.isEmpty()=true code flow.
+
+     *** Scenario 2: All window functions in the query have OVER() clause with a PARTITION BY ***
+
+     In this scenario, we need to process rows for each PARTITION BY group together, but we can batch multiple PARTITION BY keys into the same RAC before passing it to the operators for processing.
+     Batching is fine since the operators list would have the required NaivePartitioningOperatorFactory to segregate each PARTITION BY group during the processing.
+
+     The flow for this scenario can be summarised as following:
+     1. Frame Reading and Cursor Initialization: We start by reading a frame from the inputChannel and initializing frameCursor to iterate over the rows in that frame.
+     2. Row Comparison: For each row in the frame, we decide whether it belongs to the same PARTITION BY group as the previous row.
+                        This is determined by comparePartitionKeys() method.
+                        Please refer to the Javadoc of that method for further details and an example illustration.
+        2.1. If the PARTITION BY columns of current row matches the PARTITION BY columns of the previous row,
+             they belong to the same PARTITION BY group, and gets added to rowsToProcess.
+             If the number of total rows materialized exceed maxRowsMaterialized, we process the pending batch via processRowsUpToLastPartition() method.
+        2.2. If they don't match, then we have reached a partition boundary.
+             In this case, we update the value for lastPartitionIndex.
+     3. End of Input: If the input channel is finished, any remaining rows in rowsToProcess are processed.
+
+     *Illustration of Row Comparison step*
+
+     Let's say we have window_function() OVER (PARTITION BY A ORDER BY B) in our query, and we get 3 frames in the input channel to process.
+
+     Frame 1
+     A, B
+     1, 2
+     1, 3
+     2, 1 --> PARTITION BY key (column A) changed from 1 to 2.
+     2, 2
+
+     Frame 2
+     A, B
+     3, 1 --> PARTITION BY key (column A) changed from 2 to 3.
+     3, 2
+     3, 3
+     3, 4
+
+     Frame 3
+     A, B
+     3, 5
+     3, 6
+     4, 1 --> PARTITION BY key (column A) changed from 3 to 4.
+     4, 2
+
+     *Why batching?*
+     We batch multiple PARTITION BY keys for processing together to avoid the overhead of creating different RACs for each PARTITION BY keys, as that would be unnecessary in scenarios where we have a large number of PARTITION BY keys, but each key having a single row.
+
+     *Future thoughts: https://github.com/apache/druid/issues/16126*
+     Current approach with R&C and operators materialize a single R&C for processing. In case of data with low cardinality a single R&C might be too big to consume. Same for the case of empty OVER() clause.
+     Most of the window operations like SUM(), RANK(), RANGE() etc. can be made with 2 passes of the data. We might think to reimplement them in the MSQ way so that we do not have to materialize so much data.
      */
 
     if (partitionColumnNames.isEmpty()) {
-      // If we do not have any OVER() clause with PARTITION BY for this stage.
-      // Bring all data to a single executor for processing.
-      // Convert each frame to RAC.
-      // Concatenate all the racs to make a giant RAC.
-      // Let all operators run on the giant RAC until channel is finished.
+      // Scenario 1: Query has atleast one window function with an OVER() clause without a PARTITION BY.
       if (inputChannel.canRead()) {
         final Frame frame = inputChannel.read();
         convertRowFrameToRowsAndColumns(frame);
+        return ReturnOrAwait.runAgain();
       } else if (inputChannel.isFinished()) {
         runAllOpsOnMultipleRac(frameRowsAndCols);
         return ReturnOrAwait.returnObject(Unit.instance());
       } else {
         return ReturnOrAwait.awaitAll(inputChannels().size());
       }
-      return ReturnOrAwait.runAgain();
-    } else {
-      // Aha, you found a PARTITION BY and maybe ORDER BY TO
-      // PARTITION BY can also be on multiple keys
-      // typically the last stage would already partition and sort for you
-      // figure out frame boundaries and convert each distinct group to a rac
-      // then run the windowing operator only on each rac
-      if (frameCursor == null || frameCursor.isDone()) {
-        if (readableInputs.isEmpty()) {
-          return ReturnOrAwait.awaitAll(1);
-        } else if (inputChannel.canRead()) {
-          final Frame frame = inputChannel.read();
-          frameCursor = FrameProcessors.makeCursor(frame, frameReader);
-          final ColumnSelectorFactory frameColumnSelectorFactory = frameCursor.getColumnSelectorFactory();
-          final Supplier<Object>[] fieldSuppliers = new Supplier[frameReader.signature().size()];
-          for (int i = 0; i < fieldSuppliers.length; i++) {
-            final ColumnValueSelector<?> selector =
-                frameColumnSelectorFactory.makeColumnValueSelector(frameReader.signature().getColumnName(i));
-            fieldSuppliers[i] = selector::getObject;
+    }
 
-          }
-          rowSupplierFromFrameCursor = () -> {
-            final ResultRow row = ResultRow.create(fieldSuppliers.length);
-            for (int i = 0; i < fieldSuppliers.length; i++) {
-              row.set(i, fieldSuppliers[i].get());
-            }
-            return row;
-          };
-        } else if (inputChannel.isFinished()) {
-          // reaached end of channel
-          // if there is data remaining
-          // write it into a rac
-          // and run operators on it
-          if (!objectsOfASingleRac.isEmpty()) {
-            if (objectsOfASingleRac.size() > maxRowsMaterialized) {
-              throw new MSQException(new TooManyRowsInAWindowFault(objectsOfASingleRac.size(), maxRowsMaterialized));
-            }
-            RowsAndColumns rac = MapOfColumnsRowsAndColumns.fromResultRow(
-                objectsOfASingleRac,
-                frameReader.signature()
-            );
-            runAllOpsOnSingleRac(rac);
-            objectsOfASingleRac.clear();
-          }
-          return ReturnOrAwait.returnObject(Unit.instance());
-        } else {
-          return ReturnOrAwait.runAgain();
-        }
-      }
-      while (!frameCursor.isDone()) {
-        final ResultRow currentRow = rowSupplierFromFrameCursor.get();
-        if (outputRow == null) {
-          outputRow = currentRow;
-          objectsOfASingleRac.add(currentRow);
-        } else if (comparePartitionKeys(outputRow, currentRow, partitionColumnNames)) {
-          // if they have the same partition key
-          // keep adding them after checking
-          // guardrails
-          objectsOfASingleRac.add(currentRow);
-          if (objectsOfASingleRac.size() > maxRowsMaterialized) {
-            throw new MSQException(new TooManyRowsInAWindowFault(
-                objectsOfASingleRac.size(),
-                maxRowsMaterialized
-            ));
-          }
-        } else {
-          // key change noted
-          // create rac from the rows seen before
-          // run the operators on these rows and columns
-          // clean up the object to hold the new rows only
-          if (objectsOfASingleRac.size() > maxRowsMaterialized) {
-            throw new MSQException(new TooManyRowsInAWindowFault(
-                objectsOfASingleRac.size(),
-                maxRowsMaterialized
-            ));
-          }
-          RowsAndColumns rac = MapOfColumnsRowsAndColumns.fromResultRow(
-              objectsOfASingleRac,
-              frameReader.signature()
-          );
-          runAllOpsOnSingleRac(rac);
-          objectsOfASingleRac.clear();
-          outputRow = currentRow.copy();
-          return ReturnOrAwait.runAgain();
-        }
-        frameCursor.advance();
+    // Scenario 2: All window functions in the query have OVER() clause with a PARTITION BY
+    if (frameCursor == null || frameCursor.isDone()) {
+      if (readableInputs.isEmpty()) {
+        return ReturnOrAwait.awaitAll(1);
+      } else if (inputChannel.canRead()) {
+        final Frame frame = inputChannel.read();
+        frameCursor = FrameProcessors.makeCursor(frame, frameReader);
+        makeRowSupplierFromFrameCursor();
+      } else if (inputChannel.isFinished()) {
+        // Handle any remaining data.
+        lastPartitionIndex = rowsToProcess.size() - 1;
+        processRowsUpToLastPartition();
+        return ReturnOrAwait.returnObject(Unit.instance());
+      } else {
+        return ReturnOrAwait.runAgain();
       }
     }
-    return ReturnOrAwait.runAgain();
-  }
 
-  /**
-   * @param singleRac Use this {@link RowsAndColumns} as a single input for the operators to be run
-   */
-  private void runAllOpsOnSingleRac(RowsAndColumns singleRac)
-  {
-    Operator op = new Operator()
-    {
-      @Nullable
-      @Override
-      public Closeable goOrContinue(Closeable continuationObject, Receiver receiver)
-      {
-        receiver.push(singleRac);
-        if (singleRac.numRows() > maxRowsMaterialized) {
-          throw new MSQException(new TooManyRowsInAWindowFault(singleRac.numRows(), maxRowsMaterialized));
+    while (!frameCursor.isDone()) {
+      final ResultRow currentRow = rowSupplierFromFrameCursor.get();
+      if (outputRow == null) {
+        outputRow = currentRow;
+        rowsToProcess.add(currentRow);
+      } else if (comparePartitionKeys(outputRow, currentRow, partitionColumnNames)) {
+        // Add current row to the same batch of rows for processing.
+        rowsToProcess.add(currentRow);
+        if (rowsToProcess.size() > maxRowsMaterialized) {
+          // We don't want to materialize more than maxRowsMaterialized rows at any point in time, so process the pending batch.
+          processRowsUpToLastPartition();
         }
-        receiver.completed();
-        return null;
+        ensureMaxRowsInAWindowConstraint(rowsToProcess.size());
+      } else {
+        lastPartitionIndex = rowsToProcess.size() - 1;
+        outputRow = currentRow.copy();
+        return ReturnOrAwait.runAgain();
       }
-    };
-    runOperatorsAfterThis(op);
+      frameCursor.advance();
+    }
+    return ReturnOrAwait.runAgain();
   }
 
   /**
@@ -349,9 +280,7 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
       public Closeable goOrContinue(Closeable continuationObject, Receiver receiver)
       {
         RowsAndColumns rac = new ConcatRowsAndColumns(listOfRacs);
-        if (rac.numRows() > maxRowsMaterialized) {
-          throw new MSQException(new TooManyRowsInAWindowFault(rac.numRows(), maxRowsMaterialized));
-        }
+        ensureMaxRowsInAWindowConstraint(rac.numRows());
         receiver.push(rac);
         receiver.completed();
         return null;
@@ -496,12 +425,7 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
         null
     );
     // check if existing + newly added rows exceed guardrails
-    if (frameRowsAndCols.size() + ldrc.numRows() > maxRowsMaterialized) {
-      throw new MSQException(new TooManyRowsInAWindowFault(
-          frameRowsAndCols.size() + ldrc.numRows(),
-          maxRowsMaterialized
-      ));
-    }
+    ensureMaxRowsInAWindowConstraint(frameRowsAndCols.size() + ldrc.numRows());
     frameRowsAndCols.add(ldrc);
   }
 
@@ -533,4 +457,57 @@ public class WindowOperatorQueryFrameProcessor implements FrameProcessor<Object>
     }
     return match == partitionColumnNames.size();
   }
+
+  private void makeRowSupplierFromFrameCursor()
+  {
+    final ColumnSelectorFactory frameColumnSelectorFactory = frameCursor.getColumnSelectorFactory();
+    final Supplier<Object>[] fieldSuppliers = new Supplier[frameReader.signature().size()];
+    for (int i = 0; i < fieldSuppliers.length; i++) {
+      final ColumnValueSelector<?> selector =
+          frameColumnSelectorFactory.makeColumnValueSelector(frameReader.signature().getColumnName(i));
+      fieldSuppliers[i] = selector::getObject;
+    }
+    rowSupplierFromFrameCursor = () -> {
+      final ResultRow row = ResultRow.create(fieldSuppliers.length);
+      for (int i = 0; i < fieldSuppliers.length; i++) {
+        row.set(i, fieldSuppliers[i].get());
+      }
+      return row;
+    };
+  }
+
+  /**
+   * Process rows from rowsToProcess[0, lastPartitionIndex].
+   */
+  private void processRowsUpToLastPartition()
+  {
+    if (lastPartitionIndex == -1) {
+      return;
+    }
+
+    RowsAndColumns singleRac = MapOfColumnsRowsAndColumns.fromResultRowTillIndex(
+        rowsToProcess,
+        frameReader.signature(),
+        lastPartitionIndex
+    );
+    ArrayList<RowsAndColumns> rowsAndColumns = new ArrayList<>();
+    rowsAndColumns.add(singleRac);
+    runAllOpsOnMultipleRac(rowsAndColumns);
+
+    // Remove elements in the range [0, lastPartitionIndex] from the list.
+    // The call to list.subList(a, b).clear() deletes the elements in the range [a, b - 1],
+    // causing the remaining elements to shift and start from index 0.
+    rowsToProcess.subList(0, lastPartitionIndex + 1).clear();
+    lastPartitionIndex = -1;
+  }
+
+  private void ensureMaxRowsInAWindowConstraint(int numRowsInWindow)
+  {
+    if (numRowsInWindow > maxRowsMaterialized) {
+      throw new MSQException(new TooManyRowsInAWindowFault(
+          numRowsInWindow,
+          maxRowsMaterialized
+      ));
+    }
+  }
 }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/FrameProcessorTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/FrameProcessorTestBase.java
new file mode 100644
index 00000000000..75168e07ad8
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/FrameProcessorTestBase.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.querykit;
+
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.druid.frame.Frame;
+import org.apache.druid.frame.FrameType;
+import org.apache.druid.frame.channel.BlockingQueueFrameChannel;
+import org.apache.druid.frame.key.KeyColumn;
+import org.apache.druid.frame.processor.FrameProcessorExecutor;
+import org.apache.druid.frame.read.FrameReader;
+import org.apache.druid.frame.segment.FrameStorageAdapter;
+import org.apache.druid.frame.testutil.FrameSequenceBuilder;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.concurrent.Execs;
+import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.msq.input.ReadableInput;
+import org.apache.druid.msq.kernel.StageId;
+import org.apache.druid.msq.kernel.StagePartition;
+import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class FrameProcessorTestBase extends InitializedNullHandlingTest
+{
+  protected static final StagePartition STAGE_PARTITION = new StagePartition(new StageId("q", 0), 0);
+
+  protected FrameProcessorExecutor exec;
+
+  @Before
+  public void setUp()
+  {
+    exec = new FrameProcessorExecutor(MoreExecutors.listeningDecorator(Execs.singleThreaded("test-exec")));
+  }
+
+  @After
+  public void tearDown() throws Exception
+  {
+    exec.getExecutorService().shutdownNow();
+    exec.getExecutorService().awaitTermination(10, TimeUnit.MINUTES);
+  }
+
+  protected ReadableInput makeChannelFromAdapter(
+      final StorageAdapter adapter,
+      final List<KeyColumn> keyColumns,
+      int rowsPerInputFrame
+  ) throws IOException
+  {
+    // Create a single, sorted frame.
+    final FrameSequenceBuilder singleFrameBuilder =
+        FrameSequenceBuilder.fromAdapter(adapter)
+                            .frameType(FrameType.ROW_BASED)
+                            .maxRowsPerFrame(Integer.MAX_VALUE)
+                            .sortBy(keyColumns);
+
+    final RowSignature signature = singleFrameBuilder.signature();
+    final Frame frame = Iterables.getOnlyElement(singleFrameBuilder.frames().toList());
+
+    // Split it up into frames that match rowsPerFrame. Set max size enough to hold all rows we might ever want to use.
+    final BlockingQueueFrameChannel channel = new BlockingQueueFrameChannel(10_000);
+
+    final FrameReader frameReader = FrameReader.create(signature);
+
+    final FrameSequenceBuilder frameSequenceBuilder =
+        FrameSequenceBuilder.fromAdapter(new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY))
+                            .frameType(FrameType.ROW_BASED)
+                            .maxRowsPerFrame(rowsPerInputFrame);
+
+    final Sequence<Frame> frames = frameSequenceBuilder.frames();
+    frames.forEach(
+        f -> {
+          try {
+            channel.writable().write(f);
+          }
+          catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        }
+    );
+
+    channel.writable().close();
+    return ReadableInput.channel(channel.readable(), FrameReader.create(signature), STAGE_PARTITION);
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessorTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessorTest.java
new file mode 100644
index 00000000000..0b9b73facde
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessorTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.querykit;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.frame.allocation.HeapMemoryAllocator;
+import org.apache.druid.frame.allocation.SingleMemoryAllocatorFactory;
+import org.apache.druid.frame.channel.BlockingQueueFrameChannel;
+import org.apache.druid.frame.key.KeyColumn;
+import org.apache.druid.frame.read.FrameReader;
+import org.apache.druid.frame.testutil.FrameTestUtil;
+import org.apache.druid.frame.write.FrameWriterFactory;
+import org.apache.druid.frame.write.FrameWriters;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.guava.Sequences;
+import org.apache.druid.msq.counters.ChannelCounters;
+import org.apache.druid.msq.indexing.CountingWritableFrameChannel;
+import org.apache.druid.msq.indexing.error.MSQException;
+import org.apache.druid.msq.indexing.error.TooManyRowsInAWindowFault;
+import org.apache.druid.msq.input.ReadableInput;
+import org.apache.druid.msq.test.LimitedFrameWriterFactory;
+import org.apache.druid.query.Druids;
+import org.apache.druid.query.QueryDataSource;
+import org.apache.druid.query.TableDataSource;
+import org.apache.druid.query.operator.WindowOperatorQuery;
+import org.apache.druid.query.operator.window.WindowOperatorFactory;
+import org.apache.druid.query.operator.window.ranking.WindowRowNumberProcessor;
+import org.apache.druid.query.scan.ScanQuery;
+import org.apache.druid.query.spec.LegacySegmentSpec;
+import org.apache.druid.segment.RowBasedSegment;
+import org.apache.druid.segment.StorageAdapter;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.timeline.SegmentId;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.MatcherAssert;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class WindowOperatorQueryFrameProcessorTest extends FrameProcessorTestBase
+{
+  @Test
+  public void testBatchingOfPartitionByKeys_singleBatch() throws Exception
+  {
+    // With maxRowsMaterialized=100, we will get 1 frame:
+    // [1, 1, 2, 2, 2, 3, 3]
+    validateBatching(100, 1);
+  }
+
+  @Test
+  public void testBatchingOfPartitionByKeys_multipleBatches_1() throws Exception
+  {
+    // With maxRowsMaterialized=5, we will get 2 frames:
+    // [1, 1, 2, 2, 2]
+    // [3, 3]
+    validateBatching(5, 2);
+  }
+
+  @Test
+  public void testBatchingOfPartitionByKeys_multipleBatches_2() throws Exception
+  {
+    // With maxRowsMaterialized=4, we will get 3 frames:
+    // [1, 1]
+    // [2, 2, 2]
+    // [3, 3]
+    validateBatching(4, 3);
+  }
+
+  @Test
+  public void testBatchingOfPartitionByKeys_TooManyRowsInAWindowFault()
+  {
+    final RuntimeException e = Assert.assertThrows(
+        RuntimeException.class,
+        () -> validateBatching(2, 3)
+    );
+    MatcherAssert.assertThat(
+        ((MSQException) e.getCause().getCause()).getFault(),
+        CoreMatchers.instanceOf(TooManyRowsInAWindowFault.class)
+    );
+    Assert.assertTrue(e.getMessage().contains("TooManyRowsInAWindow: Too many rows in a window (requested = 3, max = 2)"));
+  }
+
+  public void validateBatching(int maxRowsMaterialized, int numFramesWritten) throws Exception
+  {
+    final ReadableInput factChannel = buildWindowTestInputChannel();
+
+    RowSignature inputSignature = RowSignature.builder()
+                                              .add("cityName", ColumnType.STRING)
+                                              .add("added", ColumnType.LONG)
+                                              .build();
+
+    FrameReader frameReader = FrameReader.create(inputSignature);
+
+    RowSignature outputSignature = RowSignature.builder()
+                                               .addAll(inputSignature)
+                                               .add("w0", ColumnType.LONG)
+                                               .build();
+
+    final BlockingQueueFrameChannel outputChannel = BlockingQueueFrameChannel.minimal();
+    ChannelCounters channelCounters = new ChannelCounters();
+    final CountingWritableFrameChannel countingWritableFrameChannel = new CountingWritableFrameChannel(
+        outputChannel.writable(),
+        channelCounters,
+        0
+    );
+
+    final WindowOperatorQuery query = new WindowOperatorQuery(
+        new QueryDataSource(
+            Druids.newScanQueryBuilder()
+                     .dataSource(new TableDataSource("test"))
+                     .intervals(new LegacySegmentSpec(Intervals.ETERNITY))
+                     .columns("cityName", "added")
+                     .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
+                     .context(new HashMap<>())
+                     .build()),
+        new LegacySegmentSpec(Intervals.ETERNITY),
+        new HashMap<>(),
+        outputSignature,
+        ImmutableList.of(
+            new WindowOperatorFactory(new WindowRowNumberProcessor("w0"))
+        ),
+        ImmutableList.of()
+    );
+
+    // Limit output frames to 1 row to ensure we test edge cases
+    final FrameWriterFactory frameWriterFactory = new LimitedFrameWriterFactory(
+        FrameWriters.makeRowBasedFrameWriterFactory(
+            new SingleMemoryAllocatorFactory(HeapMemoryAllocator.unlimited()),
+            outputSignature,
+            Collections.emptyList(),
+            false
+        ),
+        100
+    );
+
+    final WindowOperatorQueryFrameProcessor processor = new WindowOperatorQueryFrameProcessor(
+        query,
+        factChannel.getChannel(),
+        countingWritableFrameChannel,
+        frameWriterFactory,
+        frameReader,
+        new ObjectMapper(),
+        ImmutableList.of(
+            new WindowOperatorFactory(new WindowRowNumberProcessor("w0"))
+        ),
+        inputSignature,
+        maxRowsMaterialized,
+        ImmutableList.of("added")
+    );
+
+    exec.runFully(processor, null);
+
+    final Sequence<List<Object>> rowsFromProcessor = FrameTestUtil.readRowsFromFrameChannel(
+        outputChannel.readable(),
+        FrameReader.create(outputSignature)
+    );
+
+    final List<List<Object>> rows = rowsFromProcessor.toList();
+
+    long actualNumFrames = Arrays.stream(channelCounters.snapshot().getFrames()).findFirst().getAsLong();
+    Assert.assertEquals(numFramesWritten, actualNumFrames);
+    Assert.assertEquals(7, rows.size());
+  }
+
+  private ReadableInput buildWindowTestInputChannel() throws IOException
+  {
+    RowSignature inputSignature = RowSignature.builder()
+                                              .add("cityName", ColumnType.STRING)
+                                              .add("added", ColumnType.LONG)
+                                              .build();
+
+    List<Map<String, Object>> rows = ImmutableList.of(
+        ImmutableMap.of("added", 1L, "cityName", "city1"),
+        ImmutableMap.of("added", 1L, "cityName", "city2"),
+        ImmutableMap.of("added", 2L, "cityName", "city3"),
+        ImmutableMap.of("added", 2L, "cityName", "city4"),
+        ImmutableMap.of("added", 2L, "cityName", "city5"),
+        ImmutableMap.of("added", 3L, "cityName", "city6"),
+        ImmutableMap.of("added", 3L, "cityName", "city7")
+    );
+
+    return makeChannelFromRows(rows, inputSignature, Collections.emptyList());
+  }
+
+  private ReadableInput makeChannelFromRows(
+      List<Map<String, Object>> rows,
+      RowSignature signature,
+      List<KeyColumn> keyColumns
+  ) throws IOException
+  {
+    RowBasedSegment<Map<String, Object>> segment = new RowBasedSegment<>(
+        SegmentId.dummy("test"),
+        Sequences.simple(rows),
+        columnName -> m -> m.get(columnName),
+        signature
+    );
+
+    return makeChannelFromAdapter(segment.asStorageAdapter(), keyColumns);
+  }
+
+  private ReadableInput makeChannelFromAdapter(
+      final StorageAdapter adapter,
+      final List<KeyColumn> keyColumns
+  ) throws IOException
+  {
+    return makeChannelFromAdapter(adapter, keyColumns, 1000);
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/common/SortMergeJoinFrameProcessorTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/common/SortMergeJoinFrameProcessorTest.java
index 0094aeb7369..4a8ec435873 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/common/SortMergeJoinFrameProcessorTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/common/SortMergeJoinFrameProcessorTest.java
@@ -21,15 +21,11 @@ package org.apache.druid.msq.querykit.common;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.primitives.Ints;
 import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.MoreExecutors;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.common.guava.FutureUtils;
-import org.apache.druid.frame.Frame;
-import org.apache.druid.frame.FrameType;
 import org.apache.druid.frame.allocation.ArenaMemoryAllocator;
 import org.apache.druid.frame.allocation.SingleMemoryAllocatorFactory;
 import org.apache.druid.frame.channel.BlockingQueueFrameChannel;
@@ -37,23 +33,17 @@ import org.apache.druid.frame.channel.ReadableFrameChannel;
 import org.apache.druid.frame.channel.ReadableNilFrameChannel;
 import org.apache.druid.frame.key.KeyColumn;
 import org.apache.druid.frame.key.KeyOrder;
-import org.apache.druid.frame.processor.FrameProcessorExecutor;
 import org.apache.druid.frame.read.FrameReader;
-import org.apache.druid.frame.segment.FrameStorageAdapter;
-import org.apache.druid.frame.testutil.FrameSequenceBuilder;
 import org.apache.druid.frame.testutil.FrameTestUtil;
 import org.apache.druid.frame.write.FrameWriterFactory;
 import org.apache.druid.frame.write.FrameWriters;
-import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Unit;
-import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
 import org.apache.druid.msq.indexing.error.MSQException;
 import org.apache.druid.msq.indexing.error.TooManyRowsWithSameKeyFault;
 import org.apache.druid.msq.input.ReadableInput;
-import org.apache.druid.msq.kernel.StageId;
-import org.apache.druid.msq.kernel.StagePartition;
+import org.apache.druid.msq.querykit.FrameProcessorTestBase;
 import org.apache.druid.msq.test.LimitedFrameWriterFactory;
 import org.apache.druid.segment.RowBasedSegment;
 import org.apache.druid.segment.StorageAdapter;
@@ -61,15 +51,12 @@ import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.join.JoinTestHelper;
 import org.apache.druid.segment.join.JoinType;
-import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.apache.druid.timeline.SegmentId;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
-import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
-import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -82,19 +69,15 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 
 @RunWith(Parameterized.class)
-public class SortMergeJoinFrameProcessorTest extends InitializedNullHandlingTest
+public class SortMergeJoinFrameProcessorTest extends FrameProcessorTestBase
 {
-  private static final StagePartition STAGE_PARTITION = new StagePartition(new StageId("q", 0), 0);
   private static final long MAX_BUFFERED_BYTES = 10_000_000;
 
   private final int rowsPerInputFrame;
   private final int rowsPerOutputFrame;
 
-  private FrameProcessorExecutor exec;
-
   @Rule
   public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
@@ -118,19 +101,6 @@ public class SortMergeJoinFrameProcessorTest extends InitializedNullHandlingTest
     return constructors;
   }
 
-  @Before
-  public void setUp()
-  {
-    exec = new FrameProcessorExecutor(MoreExecutors.listeningDecorator(Execs.singleThreaded("test-exec")));
-  }
-
-  @After
-  public void tearDown() throws Exception
-  {
-    exec.getExecutorService().shutdownNow();
-    exec.getExecutorService().awaitTermination(10, TimeUnit.MINUTES);
-  }
-
   @Test
   public void testLeftJoinEmptyLeftSide() throws Exception
   {
@@ -1531,40 +1501,7 @@ public class SortMergeJoinFrameProcessorTest extends InitializedNullHandlingTest
       final List<KeyColumn> keyColumns
   ) throws IOException
   {
-    // Create a single, sorted frame.
-    final FrameSequenceBuilder singleFrameBuilder =
-        FrameSequenceBuilder.fromAdapter(adapter)
-                            .frameType(FrameType.ROW_BASED)
-                            .maxRowsPerFrame(Integer.MAX_VALUE)
-                            .sortBy(keyColumns);
-
-    final RowSignature signature = singleFrameBuilder.signature();
-    final Frame frame = Iterables.getOnlyElement(singleFrameBuilder.frames().toList());
-
-    // Split it up into frames that match rowsPerFrame. Set max size enough to hold all rows we might ever want to use.
-    final BlockingQueueFrameChannel channel = new BlockingQueueFrameChannel(10_000);
-
-    final FrameReader frameReader = FrameReader.create(signature);
-
-    final FrameSequenceBuilder frameSequenceBuilder =
-        FrameSequenceBuilder.fromAdapter(new FrameStorageAdapter(frame, frameReader, Intervals.ETERNITY))
-                            .frameType(FrameType.ROW_BASED)
-                            .maxRowsPerFrame(rowsPerInputFrame);
-
-    final Sequence<Frame> frames = frameSequenceBuilder.frames();
-    frames.forEach(
-        f -> {
-          try {
-            channel.writable().write(f);
-          }
-          catch (IOException e) {
-            throw new RuntimeException(e);
-          }
-        }
-    );
-
-    channel.writable().close();
-    return ReadableInput.channel(channel.readable(), FrameReader.create(signature), STAGE_PARTITION);
+    return makeChannelFromAdapter(adapter, keyColumns, rowsPerInputFrame);
   }
 
   private FrameWriterFactory makeFrameWriterFactory(final RowSignature signature)
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/results/QueryResultsFrameProcessorTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/results/QueryResultsFrameProcessorTest.java
index d89b62977a3..e300a416705 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/results/QueryResultsFrameProcessorTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/results/QueryResultsFrameProcessorTest.java
@@ -20,54 +20,32 @@
 package org.apache.druid.msq.querykit.results;
 
 import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.MoreExecutors;
 import org.apache.druid.frame.Frame;
 import org.apache.druid.frame.FrameType;
 import org.apache.druid.frame.allocation.ArenaMemoryAllocator;
 import org.apache.druid.frame.channel.BlockingQueueFrameChannel;
 import org.apache.druid.frame.channel.WritableFrameChannel;
-import org.apache.druid.frame.processor.FrameProcessorExecutor;
 import org.apache.druid.frame.read.FrameReader;
 import org.apache.druid.frame.testutil.FrameSequenceBuilder;
 import org.apache.druid.frame.testutil.FrameTestUtil;
 import org.apache.druid.java.util.common.Unit;
-import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.msq.input.ReadableInput;
 import org.apache.druid.msq.kernel.StageId;
 import org.apache.druid.msq.kernel.StagePartition;
+import org.apache.druid.msq.querykit.FrameProcessorTestBase;
 import org.apache.druid.segment.TestIndex;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter;
-import org.apache.druid.testing.InitializedNullHandlingTest;
-import org.junit.After;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
 
-public class QueryResultsFrameProcessorTest extends InitializedNullHandlingTest
+public class QueryResultsFrameProcessorTest extends FrameProcessorTestBase
 {
-  private FrameProcessorExecutor exec;
-
-  @Before
-  public void setUp()
-  {
-    exec = new FrameProcessorExecutor(MoreExecutors.listeningDecorator(Execs.singleThreaded("test-exec")));
-  }
-
-  @After
-  public void tearDown() throws Exception
-  {
-    exec.getExecutorService().shutdownNow();
-    exec.getExecutorService().awaitTermination(10, TimeUnit.MINUTES);
-  }
-
-
   @Test
   public void sanityTest() throws ExecutionException, InterruptedException, IOException
   {
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessorTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessorTest.java
index e3272cbcee7..205f3495807 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessorTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessorTest.java
@@ -20,7 +20,6 @@
 package org.apache.druid.msq.querykit.scan;
 
 import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.MoreExecutors;
 import org.apache.druid.collections.ReferenceCountingResourceHolder;
 import org.apache.druid.collections.ResourceHolder;
 import org.apache.druid.frame.Frame;
@@ -30,7 +29,6 @@ import org.apache.druid.frame.allocation.HeapMemoryAllocator;
 import org.apache.druid.frame.allocation.SingleMemoryAllocatorFactory;
 import org.apache.druid.frame.channel.BlockingQueueFrameChannel;
 import org.apache.druid.frame.channel.WritableFrameChannel;
-import org.apache.druid.frame.processor.FrameProcessorExecutor;
 import org.apache.druid.frame.read.FrameReader;
 import org.apache.druid.frame.testutil.FrameSequenceBuilder;
 import org.apache.druid.frame.testutil.FrameTestUtil;
@@ -39,11 +37,11 @@ import org.apache.druid.frame.write.FrameWriters;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Unit;
-import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.msq.input.ReadableInput;
 import org.apache.druid.msq.kernel.StageId;
 import org.apache.druid.msq.kernel.StagePartition;
+import org.apache.druid.msq.querykit.FrameProcessorTestBase;
 import org.apache.druid.msq.test.LimitedFrameWriterFactory;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.scan.ScanQuery;
@@ -51,35 +49,16 @@ import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
 import org.apache.druid.segment.TestIndex;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter;
-import org.apache.druid.testing.InitializedNullHandlingTest;
-import org.junit.After;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
-public class ScanQueryFrameProcessorTest extends InitializedNullHandlingTest
+public class ScanQueryFrameProcessorTest extends FrameProcessorTestBase
 {
-  private FrameProcessorExecutor exec;
-
-  @Before
-  public void setUp()
-  {
-    exec = new FrameProcessorExecutor(MoreExecutors.listeningDecorator(Execs.singleThreaded("test-exec")));
-  }
-
-  @After
-  public void tearDown() throws Exception
-  {
-    exec.getExecutorService().shutdownNow();
-    exec.getExecutorService().awaitTermination(10, TimeUnit.MINUTES);
-  }
-
   @Test
   public void test_runWithInputChannel() throws Exception
   {
diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/MapOfColumnsRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/MapOfColumnsRowsAndColumns.java
index 42972f9340d..29f092f6744 100644
--- a/processing/src/main/java/org/apache/druid/query/rowsandcols/MapOfColumnsRowsAndColumns.java
+++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/MapOfColumnsRowsAndColumns.java
@@ -84,11 +84,16 @@ public class MapOfColumnsRowsAndColumns implements RowsAndColumns
   }
 
   public static MapOfColumnsRowsAndColumns fromResultRow(ArrayList<ResultRow> objs, RowSignature signature)
+  {
+    return fromResultRowTillIndex(objs, signature, objs.size() - 1);
+  }
+
+  public static MapOfColumnsRowsAndColumns fromResultRowTillIndex(ArrayList<ResultRow> objs, RowSignature signature, int index)
   {
     final Builder bob = builder();
     if (!objs.isEmpty()) {
-      Object[][] columnOriented = new Object[objs.get(0).length()][objs.size()];
-      for (int i = 0; i < objs.size(); ++i) {
+      Object[][] columnOriented = new Object[objs.get(0).length()][index + 1];
+      for (int i = 0; i <= index; ++i) {
         for (int j = 0; j < objs.get(i).length(); ++j) {
           columnOriented[j][i] = objs.get(i).get(j);
         }

From 1d292c5a59cf023acdab13515fd783c44b7ffd51 Mon Sep 17 00:00:00 2001
From: 317brian <53799971+317brian@users.noreply.github.com>
Date: Wed, 28 Aug 2024 11:13:15 -0700
Subject: [PATCH 66/99] docs: fix the case of the s3 examples for msq (#16969)

---
 docs/multi-stage-query/reference.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/multi-stage-query/reference.md b/docs/multi-stage-query/reference.md
index c4345fedf81..50f07ff80b4 100644
--- a/docs/multi-stage-query/reference.md
+++ b/docs/multi-stage-query/reference.md
@@ -130,13 +130,13 @@ FROM <table>
 
 ##### S3
 
-Export results to S3 by passing the function `S3()` as an argument to the `EXTERN` function. Note that this requires the `druid-s3-extensions`.
-The `S3()` function is a Druid function that configures the connection. Arguments for `S3()` should be passed as named parameters with the value in single quotes like the following example:
+Export results to S3 by passing the function `s3()` as an argument to the `EXTERN` function. Note that this requires the `druid-s3-extensions`.
+The `s3()` function is a Druid function that configures the connection. Arguments for `s3()` should be passed as named parameters with the value in single quotes like the following example:
 
 ```sql
 INSERT INTO
   EXTERN(
-    S3(bucket => 'your_bucket', prefix => 'prefix/to/files')
+    s3(bucket => 'your_bucket', prefix => 'prefix/to/files')
   )
 AS CSV
 SELECT

From 358d06abc133bb2597230cd39cf2e0c6e43e1800 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Thu, 29 Aug 2024 09:58:15 -0700
Subject: [PATCH 67/99] Web console: expose forceSegmentSortByTime (#16967)

* no force time

* time UI

* update menus

* tweaks

* dont use bp5

* nicer values

* update snapshots

* similar engine lables

* update snaps
---
 docs/operations/web-console.md                |   2 +-
 docs/tutorials/tutorial-sql-query-view.md     |   4 +-
 web-console/src/components/index.ts           |   2 +-
 .../__snapshots__/menu-boolean.spec.tsx.snap} |  62 ++++++-
 .../menu-boolean.spec.tsx}                    |  16 +-
 .../menu-boolean.tsx}                         |  55 ++++--
 .../dimension-spec/dimension-spec.ts          |  12 +-
 .../ingestion-spec/ingestion-spec.spec.ts     |  88 ++++++++++
 .../ingestion-spec/ingestion-spec.tsx         |  18 ++
 .../query-context/query-context.tsx           |   2 +
 web-console/src/entry.scss                    |   6 +
 web-console/src/helpers/spec-conversion.ts    |   8 +
 web-console/src/utils/sampler.ts              |  12 +-
 .../views/load-data-view/load-data-view.tsx   | 143 ++++++++++++++--
 .../schema-table/schema-table.tsx             |   4 +-
 .../schema-step/schema-step.tsx               |  33 +++-
 .../sql-data-loader-view.tsx                  |  12 ++
 .../max-tasks-button.spec.tsx.snap            |  24 ++-
 .../max-tasks-button/max-tasks-button.tsx     |  31 +++-
 .../__snapshots__/run-panel.spec.tsx.snap     |   4 +-
 .../workbench-view/run-panel/run-panel.tsx    | 162 ++++++++++++------
 21 files changed, 575 insertions(+), 125 deletions(-)
 rename web-console/src/components/{menu-tristate/__snapshots__/menu-tristate.spec.tsx.snap => menu-boolean/__snapshots__/menu-boolean.spec.tsx.snap} (63%)
 rename web-console/src/components/{menu-tristate/menu-tristate.spec.tsx => menu-boolean/menu-boolean.spec.tsx} (78%)
 rename web-console/src/components/{menu-tristate/menu-tristate.tsx => menu-boolean/menu-boolean.tsx} (55%)

diff --git a/docs/operations/web-console.md b/docs/operations/web-console.md
index db25792d3e0..ef1118ebc4c 100644
--- a/docs/operations/web-console.md
+++ b/docs/operations/web-console.md
@@ -87,7 +87,7 @@ It is equivalent to the **Task** view in the **Ingestion** view with the filter
 9. The **Preview** button appears when you enter an INSERT/REPLACE query. It runs the query inline without the INSERT/REPLACE clause and with an added LIMIT to give you a preview of the data that would be ingested if you click **Run**.
 The added LIMIT makes the query run faster but provides incomplete results.
 10. The engine selector lets you choose which engine (API endpoint) to send a query to. By default, it automatically picks which endpoint to use based on an analysis of the query, but you can select a specific engine explicitly. You can also configure the engine specific context parameters from this menu.
-11. The **Max tasks** picker appears when you have the **sql-msq-task** engine selected. It lets you configure the degree of parallelism.
+11. The **Max tasks** picker appears when you have the **SQL MSQ-task** engine selected. It lets you configure the degree of parallelism.
 12. The More menu (**...**) contains the following helpful tools:
 - **Explain SQL query** shows you the logical plan returned by `EXPLAIN PLAN FOR` for a SQL query.
 - **Query history** shows you previously executed queries.
diff --git a/docs/tutorials/tutorial-sql-query-view.md b/docs/tutorials/tutorial-sql-query-view.md
index a313c7a300c..e51661f56b7 100644
--- a/docs/tutorials/tutorial-sql-query-view.md
+++ b/docs/tutorials/tutorial-sql-query-view.md
@@ -100,9 +100,9 @@ In this section you run some queries using aggregate functions and perform some
 
    ![aggregate-query](../assets/tutorial-sql-aggregate-query.png)
 
-7. Click **Engine: auto (sql-native)** to display the engine options&mdash;**native** for native (JSON-based) queries, **sql-native** for Druid SQL queries, and **sql-msq-task** for SQL-based ingestion. 
+7. Click **Engine: Auto (SQL native)** to display the engine options&mdash;**Native** for native (JSON-based) queries, **SQL native** for Druid SQL queries, and **SQL MSQ-task** for SQL-based ingestion. 
 
-   Select **auto** to let Druid select the most efficient engine based on your query input.
+   Select **Auto** to let Druid select the most efficient engine based on your query input.
 
 8. From the engine menu you can also edit the query context and turn off some query defaults. 
 
diff --git a/web-console/src/components/index.ts b/web-console/src/components/index.ts
index 42f8257b5dd..bbae88a5d43 100644
--- a/web-console/src/components/index.ts
+++ b/web-console/src/components/index.ts
@@ -37,8 +37,8 @@ export * from './json-collapse/json-collapse';
 export * from './json-input/json-input';
 export * from './learn-more/learn-more';
 export * from './loader/loader';
+export * from './menu-boolean/menu-boolean';
 export * from './menu-checkbox/menu-checkbox';
-export * from './menu-tristate/menu-tristate';
 export * from './more-button/more-button';
 export * from './plural-pair-if-needed/plural-pair-if-needed';
 export * from './popover-text/popover-text';
diff --git a/web-console/src/components/menu-tristate/__snapshots__/menu-tristate.spec.tsx.snap b/web-console/src/components/menu-boolean/__snapshots__/menu-boolean.spec.tsx.snap
similarity index 63%
rename from web-console/src/components/menu-tristate/__snapshots__/menu-tristate.spec.tsx.snap
rename to web-console/src/components/menu-boolean/__snapshots__/menu-boolean.spec.tsx.snap
index d5a0f0c2e26..3fc368ecd35 100644
--- a/web-console/src/components/menu-tristate/__snapshots__/menu-tristate.spec.tsx.snap
+++ b/web-console/src/components/menu-boolean/__snapshots__/menu-boolean.spec.tsx.snap
@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`MenuTristate matches snapshot false 1`] = `
+exports[`MenuBoolean matches snapshot false 1`] = `
 <li
   class="bp5-submenu"
   role="none"
@@ -54,7 +54,7 @@ exports[`MenuTristate matches snapshot false 1`] = `
 </li>
 `;
 
-exports[`MenuTristate matches snapshot undefined 1`] = `
+exports[`MenuBoolean matches snapshot no undefined 1`] = `
 <li
   class="bp5-submenu"
   role="none"
@@ -68,7 +68,7 @@ exports[`MenuTristate matches snapshot undefined 1`] = `
       aria-expanded="false"
       aria-haspopup="menu"
       class="bp5-menu-item menu-tristate"
-      label="auto (true)"
+      label="false"
       role="none"
       tabindex="-1"
     >
@@ -80,7 +80,61 @@ exports[`MenuTristate matches snapshot undefined 1`] = `
       <span
         class="bp5-menu-item-label"
       >
-        auto (true)
+        false
+      </span>
+      <span
+        class="bp5-icon bp5-icon-caret-right bp5-submenu-icon"
+      >
+        <svg
+          data-icon="caret-right"
+          height="16"
+          role="img"
+          viewBox="0 0 16 16"
+          width="16"
+        >
+          <title>
+            Open sub menu
+          </title>
+          <path
+            d="M220 160C220 163 218.6 165.6 216.6 167.4L216.6 167.4L136.6 237.4L136.6 237.4C134.8 239 132.6 240 130 240C124.4 240 120 235.6 120 230V90C120 84.4 124.4 80 130 80C132.6 80 134.8 81 136.6 82.6C136.6 82.6 136.6 82.6 136.6 82.6L216.6 152.6L216.6 152.6C218.6 154.4 220 157 220 160z"
+            fill-rule="evenodd"
+            style="transform-origin: center;"
+            transform="scale(0.05, -0.05) translate(-160, -160)"
+          />
+        </svg>
+      </span>
+    </a>
+  </span>
+</li>
+`;
+
+exports[`MenuBoolean matches snapshot undefined 1`] = `
+<li
+  class="bp5-submenu"
+  role="none"
+>
+  <span
+    class="bp5-popover-target"
+    role="menuitem"
+    tabindex="0"
+  >
+    <a
+      aria-expanded="false"
+      aria-haspopup="menu"
+      class="bp5-menu-item menu-tristate"
+      label="Auto (true)"
+      role="none"
+      tabindex="-1"
+    >
+      <div
+        class="bp5-text-overflow-ellipsis bp5-fill"
+      >
+        hello
+      </div>
+      <span
+        class="bp5-menu-item-label"
+      >
+        Auto (true)
       </span>
       <span
         class="bp5-icon bp5-icon-caret-right bp5-submenu-icon"
diff --git a/web-console/src/components/menu-tristate/menu-tristate.spec.tsx b/web-console/src/components/menu-boolean/menu-boolean.spec.tsx
similarity index 78%
rename from web-console/src/components/menu-tristate/menu-tristate.spec.tsx
rename to web-console/src/components/menu-boolean/menu-boolean.spec.tsx
index 40fbc54f0db..f43c4c5dba1 100644
--- a/web-console/src/components/menu-tristate/menu-tristate.spec.tsx
+++ b/web-console/src/components/menu-boolean/menu-boolean.spec.tsx
@@ -19,14 +19,15 @@
 import { render } from '@testing-library/react';
 import React from 'react';
 
-import { MenuTristate } from './menu-tristate';
+import { MenuBoolean } from './menu-boolean';
 
-describe('MenuTristate', () => {
+describe('MenuBoolean', () => {
   it('matches snapshot undefined', () => {
     const menuCheckbox = (
-      <MenuTristate
+      <MenuBoolean
         text="hello"
         value={undefined}
+        showUndefined
         undefinedEffectiveValue
         onValueChange={() => {}}
       />
@@ -37,9 +38,10 @@ describe('MenuTristate', () => {
 
   it('matches snapshot false', () => {
     const menuCheckbox = (
-      <MenuTristate
+      <MenuBoolean
         text="hello"
         value={false}
+        showUndefined
         undefinedEffectiveValue={false}
         onValueChange={() => {}}
       />
@@ -47,4 +49,10 @@ describe('MenuTristate', () => {
     const { container } = render(menuCheckbox);
     expect(container.firstChild).toMatchSnapshot();
   });
+
+  it('matches snapshot no undefined', () => {
+    const menuCheckbox = <MenuBoolean text="hello" value={false} onValueChange={() => {}} />;
+    const { container } = render(menuCheckbox);
+    expect(container.firstChild).toMatchSnapshot();
+  });
 });
diff --git a/web-console/src/components/menu-tristate/menu-tristate.tsx b/web-console/src/components/menu-boolean/menu-boolean.tsx
similarity index 55%
rename from web-console/src/components/menu-tristate/menu-tristate.tsx
rename to web-console/src/components/menu-boolean/menu-boolean.tsx
index 783cdc7e8e3..8e4713623cd 100644
--- a/web-console/src/components/menu-tristate/menu-tristate.tsx
+++ b/web-console/src/components/menu-boolean/menu-boolean.tsx
@@ -19,50 +19,71 @@
 import type { MenuItemProps } from '@blueprintjs/core';
 import { MenuItem } from '@blueprintjs/core';
 import classNames from 'classnames';
+import type { ReactNode } from 'react';
 import React from 'react';
 
 import { tickIcon } from '../../utils';
 
-export interface MenuTristateProps extends Omit<MenuItemProps, 'label'> {
-  value: boolean | undefined;
-  onValueChange(value: boolean | undefined): void;
-  undefinedLabel?: string;
-  undefinedEffectiveValue?: boolean;
+export type TrueFalseUndefined = 'true' | 'false' | 'undefined';
+
+function toKey(value: boolean | undefined) {
+  return String(value) as TrueFalseUndefined;
 }
 
-export function MenuTristate(props: MenuTristateProps) {
+const DEFAULT_OPTIONS_TEXT: Partial<Record<TrueFalseUndefined, string>> = { undefined: 'Auto' };
+
+export const ENABLE_DISABLE_OPTIONS_TEXT: Partial<Record<TrueFalseUndefined, string>> = {
+  true: 'Enable',
+  false: 'Disable',
+  undefined: 'Auto',
+};
+
+export interface MenuBooleanProps extends Omit<MenuItemProps, 'label'> {
+  value: boolean | undefined;
+  onValueChange(value: boolean | undefined): void;
+  showUndefined?: boolean;
+  undefinedEffectiveValue?: boolean;
+  optionsText?: Partial<Record<TrueFalseUndefined, string>>;
+  optionsLabelElement?: Partial<Record<TrueFalseUndefined, ReactNode>>;
+}
+
+export function MenuBoolean(props: MenuBooleanProps) {
   const {
     value,
     onValueChange,
-    undefinedLabel,
+    showUndefined,
     undefinedEffectiveValue,
     className,
     shouldDismissPopover,
+    optionsText = DEFAULT_OPTIONS_TEXT,
+    optionsLabelElement = {},
     ...rest
   } = props;
+  const effectiveValue = showUndefined ? value : value ?? undefinedEffectiveValue;
   const shouldDismiss = shouldDismissPopover ?? false;
 
   function formatValue(value: boolean | undefined): string {
-    return String(value ?? undefinedLabel ?? 'auto');
+    const s = toKey(value);
+    return optionsText[s] ?? s;
   }
 
   return (
     <MenuItem
       className={classNames('menu-tristate', className)}
       shouldDismissPopover={shouldDismiss}
-      label={
-        formatValue(value) +
-        (typeof value === 'undefined' && typeof undefinedEffectiveValue === 'boolean'
-          ? ` (${undefinedEffectiveValue})`
-          : '')
-      }
+      label={`${formatValue(effectiveValue)}${
+        typeof effectiveValue === 'undefined' && typeof undefinedEffectiveValue === 'boolean'
+          ? ` (${formatValue(undefinedEffectiveValue)})`
+          : ''
+      }`}
       {...rest}
     >
-      {[undefined, true, false].map((v, i) => (
+      {(showUndefined ? [undefined, true, false] : [true, false]).map(v => (
         <MenuItem
-          key={i}
-          icon={tickIcon(value === v)}
+          key={String(v)}
+          icon={tickIcon(effectiveValue === v)}
           text={formatValue(v)}
+          labelElement={optionsLabelElement[toKey(v)]}
           onClick={() => onValueChange(v)}
           shouldDismissPopover={shouldDismiss}
         />
diff --git a/web-console/src/druid-models/dimension-spec/dimension-spec.ts b/web-console/src/druid-models/dimension-spec/dimension-spec.ts
index f8d8f229dab..c0dcad2fa4d 100644
--- a/web-console/src/druid-models/dimension-spec/dimension-spec.ts
+++ b/web-console/src/druid-models/dimension-spec/dimension-spec.ts
@@ -18,9 +18,10 @@
 
 import type { Field } from '../../components';
 import { filterMap, typeIsKnown } from '../../utils';
-import type { SampleResponse } from '../../utils/sampler';
+import type { SampleResponse, TimeColumnAction } from '../../utils/sampler';
 import { getHeaderNamesFromSampleResponse } from '../../utils/sampler';
 import { guessColumnTypeFromSampleResponse } from '../ingestion-spec/ingestion-spec';
+import { TIME_COLUMN } from '../timestamp-spec/timestamp-spec';
 
 export interface DimensionsSpec {
   readonly dimensions?: (string | DimensionSpec)[];
@@ -28,6 +29,7 @@ export interface DimensionsSpec {
   readonly spatialDimensions?: any[];
   readonly includeAllDimensions?: boolean;
   readonly useSchemaDiscovery?: boolean;
+  readonly forceSegmentSortByTime?: boolean;
 }
 
 export interface DimensionSpec {
@@ -61,6 +63,7 @@ export const DIMENSION_SPEC_FIELDS: Field<DimensionSpec>[] = [
     type: 'string',
     required: true,
     suggestions: KNOWN_TYPES,
+    disabled: d => d.name === TIME_COLUMN,
   },
   {
     name: 'createBitmapIndex',
@@ -163,8 +166,13 @@ export function getDimensionSpecs(
   guessNumericStringsAsNumbers: boolean,
   forceMvdInsteadOfArray: boolean,
   hasRollup: boolean,
+  timeColumnAction: TimeColumnAction,
 ): (string | DimensionSpec)[] {
-  return filterMap(getHeaderNamesFromSampleResponse(sampleResponse, 'ignore'), h => {
+  return filterMap(getHeaderNamesFromSampleResponse(sampleResponse, timeColumnAction), h => {
+    if (h === TIME_COLUMN) {
+      return { type: 'long', name: h };
+    }
+
     const columnTypeHint = columnTypeHints[h];
     const guessedColumnType = guessColumnTypeFromSampleResponse(
       sampleResponse,
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
index d58e8c07792..1371001e980 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
@@ -22,6 +22,7 @@ import type { IngestionSpec } from './ingestion-spec';
 import {
   adjustId,
   cleanSpec,
+  DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
   guessColumnTypeFromInput,
   guessColumnTypeFromSampleResponse,
   guessKafkaInputFormat,
@@ -857,10 +858,94 @@ describe('spec utils', () => {
   });
 
   describe('updateSchemaWithSample', () => {
+    it('works with when not forcing time, arrays', () => {
+      const updateSpec = updateSchemaWithSample(
+        ingestionSpec,
+        JSON_SAMPLE,
+        false,
+        'fixed',
+        'arrays',
+        true,
+      );
+      expect(updateSpec.spec).toMatchInlineSnapshot(`
+        {
+          "dataSchema": {
+            "dataSource": "wikipedia",
+            "dimensionsSpec": {
+              "dimensions": [
+                {
+                  "name": "__time",
+                  "type": "long",
+                },
+                "user",
+                "id",
+                {
+                  "castToType": "ARRAY<STRING>",
+                  "name": "tags",
+                  "type": "auto",
+                },
+                {
+                  "castToType": "ARRAY<LONG>",
+                  "name": "nums",
+                  "type": "auto",
+                },
+              ],
+              "forceSegmentSortByTime": false,
+            },
+            "granularitySpec": {
+              "queryGranularity": "hour",
+              "rollup": true,
+              "segmentGranularity": "day",
+            },
+            "metricsSpec": [
+              {
+                "name": "count",
+                "type": "count",
+              },
+              {
+                "fieldName": "followers",
+                "name": "sum_followers",
+                "type": "longSum",
+              },
+              {
+                "fieldName": "spend",
+                "name": "sum_spend",
+                "type": "doubleSum",
+              },
+            ],
+            "timestampSpec": {
+              "column": "timestamp",
+              "format": "iso",
+            },
+          },
+          "ioConfig": {
+            "inputFormat": {
+              "type": "json",
+            },
+            "inputSource": {
+              "type": "http",
+              "uris": [
+                "https://website.com/wikipedia.json.gz",
+              ],
+            },
+            "type": "index_parallel",
+          },
+          "tuningConfig": {
+            "forceGuaranteedRollup": true,
+            "partitionsSpec": {
+              "type": "hashed",
+            },
+            "type": "index_parallel",
+          },
+        }
+      `);
+    });
+
     it('works with rollup, arrays', () => {
       const updateSpec = updateSchemaWithSample(
         ingestionSpec,
         JSON_SAMPLE,
+        DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
         'fixed',
         'arrays',
         true,
@@ -938,6 +1023,7 @@ describe('spec utils', () => {
       const updateSpec = updateSchemaWithSample(
         ingestionSpec,
         JSON_SAMPLE,
+        DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
         'fixed',
         'multi-values',
         true,
@@ -1015,6 +1101,7 @@ describe('spec utils', () => {
       const updatedSpec = updateSchemaWithSample(
         ingestionSpec,
         JSON_SAMPLE,
+        DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
         'fixed',
         'arrays',
         false,
@@ -1083,6 +1170,7 @@ describe('spec utils', () => {
       const updatedSpec = updateSchemaWithSample(
         ingestionSpec,
         JSON_SAMPLE,
+        DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
         'fixed',
         'multi-values',
         false,
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index 3a7f0ae5674..03898732bb0 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -295,6 +295,13 @@ export type SchemaMode = 'fixed' | 'string-only-discovery' | 'type-aware-discove
 
 export type ArrayMode = 'arrays' | 'multi-values';
 
+export const DEFAULT_FORCE_SEGMENT_SORT_BY_TIME = true;
+export function getForceSegmentSortByTime(spec: Partial<IngestionSpec>): boolean {
+  return (
+    deepGet(spec, 'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime') ??
+    DEFAULT_FORCE_SEGMENT_SORT_BY_TIME
+  );
+}
 export function getSchemaMode(spec: Partial<IngestionSpec>): SchemaMode {
   if (deepGet(spec, 'spec.dataSchema.dimensionsSpec.useSchemaDiscovery') === true) {
     return 'type-aware-discovery';
@@ -2744,6 +2751,7 @@ function getColumnTypeHintsFromSpec(spec: Partial<IngestionSpec>): Record<string
 export function updateSchemaWithSample(
   spec: Partial<IngestionSpec>,
   sampleResponse: SampleResponse,
+  forceSegmentSortByTime: boolean,
   schemaMode: SchemaMode,
   arrayMode: ArrayMode,
   rollup: boolean,
@@ -2756,6 +2764,15 @@ export function updateSchemaWithSample(
 
   let newSpec = spec;
 
+  newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime');
+  if (forceSegmentSortByTime !== DEFAULT_FORCE_SEGMENT_SORT_BY_TIME) {
+    newSpec = deepSet(
+      newSpec,
+      'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime',
+      forceSegmentSortByTime,
+    );
+  }
+
   switch (schemaMode) {
     case 'type-aware-discovery':
       newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.useSchemaDiscovery', true);
@@ -2784,6 +2801,7 @@ export function updateSchemaWithSample(
           guessNumericStringsAsNumbers,
           arrayMode === 'multi-values',
           rollup,
+          forceSegmentSortByTime ?? DEFAULT_FORCE_SEGMENT_SORT_BY_TIME ? 'ignore' : 'preserve',
         ),
       );
       break;
diff --git a/web-console/src/druid-models/query-context/query-context.tsx b/web-console/src/druid-models/query-context/query-context.tsx
index 4b0367c25a6..ee467577510 100644
--- a/web-console/src/druid-models/query-context/query-context.tsx
+++ b/web-console/src/druid-models/query-context/query-context.tsx
@@ -41,6 +41,7 @@ export interface QueryContext {
   failOnEmptyInsert?: boolean;
   waitUntilSegmentsLoad?: boolean;
   useConcurrentLocks?: boolean;
+  forceSegmentSortByTime?: boolean;
 
   [key: string]: any;
 }
@@ -63,6 +64,7 @@ export const DEFAULT_SERVER_QUERY_CONTEXT: QueryContext = {
   failOnEmptyInsert: false,
   waitUntilSegmentsLoad: false,
   useConcurrentLocks: false,
+  forceSegmentSortByTime: true,
 };
 
 export interface QueryWithContext {
diff --git a/web-console/src/entry.scss b/web-console/src/entry.scss
index 3893d7ccbee..1304f2aa350 100644
--- a/web-console/src/entry.scss
+++ b/web-console/src/entry.scss
@@ -63,3 +63,9 @@ body {
     width: 100%;
   }
 }
+
+// Prevent popover menus from being longer than 45% of available height, let them scroll instead
+.#{$bp-ns}-popover-content > .#{$bp-ns}-menu {
+  max-height: 47vh;
+  overflow: auto;
+}
diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts
index 2c621f2466c..00bc3d5f832 100644
--- a/web-console/src/helpers/spec-conversion.ts
+++ b/web-console/src/helpers/spec-conversion.ts
@@ -86,6 +86,14 @@ export function convertSpecToSql(spec: any): QueryWithContext {
     context.arrayIngestMode = 'array';
   }
 
+  const forceSegmentSortByTime = deepGet(
+    spec,
+    'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime',
+  );
+  if (typeof forceSegmentSortByTime !== 'undefined') {
+    context.forceSegmentSortByTime = forceSegmentSortByTime;
+  }
+
   const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
   if (indexSpec) {
     context.indexSpec = indexSpec;
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index cc9ae32b8ae..f74acf024c9 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -73,16 +73,18 @@ export interface SampleResponse {
   numRowsRead: number;
 }
 
+export type TimeColumnAction = 'preserve' | 'ignore' | 'ignoreIfZero';
+
 export function getHeaderNamesFromSampleResponse(
   sampleResponse: SampleResponse,
-  timeColumnAction: 'preserve' | 'ignore' | 'ignoreIfZero' = 'preserve',
+  timeColumnAction: TimeColumnAction = 'preserve',
 ): string[] {
   return getHeaderFromSampleResponse(sampleResponse, timeColumnAction).map(s => s.name);
 }
 
 export function getHeaderFromSampleResponse(
   sampleResponse: SampleResponse,
-  timeColumnAction: 'preserve' | 'ignore' | 'ignoreIfZero' = 'preserve',
+  timeColumnAction: TimeColumnAction = 'preserve',
 ): { name: string; type: string }[] {
   const ignoreTimeColumn =
     timeColumnAction === 'ignore' ||
@@ -462,13 +464,17 @@ export async function sampleForTimestamp(
 export async function sampleForTransform(
   spec: Partial<IngestionSpec>,
   cacheRows: CacheRows,
+  forceSegmentSortByTime: boolean,
 ): Promise<SampleResponse> {
   const samplerType = getSpecType(spec);
   const timestampSpec: TimestampSpec = deepGet(spec, 'spec.dataSchema.timestampSpec');
   const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || [];
 
   // Extra step to simulate auto-detecting dimension with transforms
-  let specialDimensionSpec: DimensionsSpec = { useSchemaDiscovery: true };
+  let specialDimensionSpec: DimensionsSpec = {
+    useSchemaDiscovery: true,
+    forceSegmentSortByTime,
+  };
   if (transforms && transforms.length) {
     const sampleSpecHack: SampleSpec = {
       type: samplerType,
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index 968b53888c6..42cfa25a7b7 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -83,6 +83,7 @@ import {
   computeFlattenPathsForData,
   CONSTANT_TIMESTAMP_SPEC,
   CONSTANT_TIMESTAMP_SPEC_FIELDS,
+  DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
   DIMENSION_SPEC_FIELDS,
   fillDataSourceNameIfNeeded,
   fillInputFormatIfNeeded,
@@ -92,6 +93,7 @@ import {
   getArrayMode,
   getDimensionSpecName,
   getFlattenSpec,
+  getForceSegmentSortByTime,
   getIngestionComboType,
   getIngestionImage,
   getIngestionTitle,
@@ -311,7 +313,14 @@ function initializeSchemaWithSampleIfNeeded(
   sample: SampleResponse,
 ): Partial<IngestionSpec> {
   if (deepGet(spec, 'spec.dataSchema.dimensionsSpec')) return spec;
-  return updateSchemaWithSample(spec, sample, 'fixed', 'multi-values', getRollup(spec, false));
+  return updateSchemaWithSample(
+    spec,
+    sample,
+    DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
+    'fixed',
+    'multi-values',
+    getRollup(spec, false),
+  );
 }
 
 type Step =
@@ -394,6 +403,7 @@ export interface LoadDataViewState {
   continueToSpec: boolean;
   showResetConfirm: boolean;
   newRollup?: boolean;
+  newForceSegmentSortByTime?: boolean;
   newSchemaMode?: SchemaMode;
   newArrayMode?: ArrayMode;
 
@@ -1965,7 +1975,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
     let sampleResponse: SampleResponse;
     try {
-      sampleResponse = await sampleForTransform(spec, cacheRows);
+      sampleResponse = await sampleForTransform(
+        spec,
+        cacheRows,
+        DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
+      );
     } catch (e) {
       this.setState(({ transformQueryState }) => ({
         transformQueryState: new QueryState({
@@ -2359,6 +2373,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     const somethingSelected = Boolean(
       selectedAutoDimension || selectedDimensionSpec || selectedMetricSpec,
     );
+    const forceSegmentSortByTime = getForceSegmentSortByTime(spec);
     const schemaMode = getSchemaMode(spec);
     const arrayMode = getArrayMode(spec);
 
@@ -2404,6 +2419,39 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           <SchemaMessage schemaMode={schemaMode} />
           {!somethingSelected && (
             <>
+              <FormGroupWithInfo
+                inlineInfo
+                info={
+                  <PopoverText>
+                    <p>
+                      When set to true (the default), segments created by the ingestion job are
+                      sorted by <Code>{'{__time, dimensions[0], dimensions[1], ...}'}</Code>. When
+                      set to false, segments created by the ingestion job are sorted by{' '}
+                      <Code>{'{dimensions[0], dimensions[1], ...}'}</Code>. To include{' '}
+                      <Code>__time</Code> in the sort order when this parameter is set to{' '}
+                      <Code>false</Code>, you must include a dimension named <Code>__time</Code>{' '}
+                      with type <Code>long</Code> explicitly in the `dimensions` list.
+                    </p>
+                    <p>
+                      Setting this to `false` is an experimental feature; see
+                      <ExternalLink href={`${getLink('DOCS')}/ingestion/partitioning/#sorting`}>
+                        Sorting
+                      </ExternalLink>{' '}
+                      for details.
+                    </p>
+                  </PopoverText>
+                }
+              >
+                <Switch
+                  checked={forceSegmentSortByTime}
+                  onChange={() =>
+                    this.setState({
+                      newForceSegmentSortByTime: !forceSegmentSortByTime,
+                    })
+                  }
+                  label="Force segment sort by time"
+                />
+              </FormGroupWithInfo>
               <FormGroupWithInfo
                 inlineInfo
                 info={
@@ -2563,7 +2611,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               {schemaToolsMenu && (
                 <FormGroup>
                   <Popover content={schemaToolsMenu}>
-                    <Button icon={IconNames.BUILD} />
+                    <Button icon={IconNames.BUILD} text="Tools" />
                   </Popover>
                 </FormGroup>
               )}
@@ -2572,6 +2620,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           {this.renderAutoDimensionControls()}
           {this.renderDimensionSpecControls()}
           {this.renderMetricSpecControls()}
+          {this.renderChangeForceSegmentSortByTime()}
           {this.renderChangeRollupAction()}
           {this.renderChangeSchemaModeAction()}
           {this.renderChangeArrayModeAction()}
@@ -2660,6 +2709,44 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     });
   };
 
+  renderChangeForceSegmentSortByTime() {
+    const { newForceSegmentSortByTime, spec, cacheRows } = this.state;
+    if (typeof newForceSegmentSortByTime === 'undefined' || !cacheRows) return;
+
+    return (
+      <AsyncActionDialog
+        action={async () => {
+          const sampleResponse = await sampleForTransform(
+            spec,
+            cacheRows,
+            newForceSegmentSortByTime,
+          );
+          this.updateSpec(
+            updateSchemaWithSample(
+              spec,
+              sampleResponse,
+              newForceSegmentSortByTime,
+              getSchemaMode(spec),
+              getArrayMode(spec),
+              getRollup(spec),
+              true,
+            ),
+          );
+        }}
+        confirmButtonText={`Yes - ${
+          newForceSegmentSortByTime ? 'force time to be first' : "don't force time to be first"
+        }`}
+        successText={`forceSegmentSortByTime was set to ${newForceSegmentSortByTime}.`}
+        failText="Could change rollup"
+        intent={Intent.WARNING}
+        onClose={() => this.setState({ newForceSegmentSortByTime: undefined })}
+      >
+        <p>{`Are you sure you want to set forceSegmentSortByTime to ${newForceSegmentSortByTime}?`}</p>
+        <p>Making this change will reset any work you have done in this section.</p>
+      </AsyncActionDialog>
+    );
+  }
+
   renderChangeRollupAction() {
     const { newRollup, spec, cacheRows } = this.state;
     if (typeof newRollup === 'undefined' || !cacheRows) return;
@@ -2667,11 +2754,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     return (
       <AsyncActionDialog
         action={async () => {
-          const sampleResponse = await sampleForTransform(spec, cacheRows);
+          const sampleResponse = await sampleForTransform(
+            spec,
+            cacheRows,
+            getForceSegmentSortByTime(spec),
+          );
           this.updateSpec(
             updateSchemaWithSample(
               spec,
               sampleResponse,
+              getForceSegmentSortByTime(spec),
               getSchemaMode(spec),
               getArrayMode(spec),
               newRollup,
@@ -2699,11 +2791,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     return (
       <AsyncActionDialog
         action={async () => {
-          const sampleResponse = await sampleForTransform(spec, cacheRows);
+          const sampleResponse = await sampleForTransform(
+            spec,
+            cacheRows,
+            getForceSegmentSortByTime(spec),
+          );
           this.updateSpec(
             updateSchemaWithSample(
               spec,
               sampleResponse,
+              getForceSegmentSortByTime(spec),
               newSchemaMode,
               getArrayMode(spec),
               getRollup(spec),
@@ -2766,11 +2863,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     return (
       <AsyncActionDialog
         action={async () => {
-          const sampleResponse = await sampleForTransform(spec, cacheRows);
+          const sampleResponse = await sampleForTransform(
+            spec,
+            cacheRows,
+            getForceSegmentSortByTime(spec),
+          );
           this.updateSpec(
             updateSchemaWithSample(
               spec,
               sampleResponse,
+              getForceSegmentSortByTime(spec),
               getSchemaMode(spec),
               newArrayMode,
               getRollup(spec),
@@ -2829,6 +2931,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   renderDimensionSpecControls() {
     const { spec, selectedDimensionSpec } = this.state;
     if (!selectedDimensionSpec) return;
+    const selectedTime = selectedDimensionSpec.value.name === TIME_COLUMN;
     const schemaMode = getSchemaMode(spec);
 
     const dimensions = deepGet(spec, `spec.dataSchema.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
@@ -2913,7 +3016,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           )
         }
         showDelete={selectedDimensionSpec.index !== -1}
-        disableDelete={dimensions.length <= 1}
+        disableDelete={dimensions.length <= 1 || selectedTime}
         onDelete={() =>
           this.updateSpec(
             deepDelete(
@@ -2934,18 +3037,20 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             </Popover>
           </FormGroup>
         )}
-        {selectedDimensionSpec.index !== -1 && deepGet(spec, 'spec.dataSchema.metricsSpec') && (
-          <FormGroup>
-            <Popover content={convertToMetricMenu}>
-              <Button
-                icon={IconNames.EXCHANGE}
-                text="Convert to metric"
-                rightIcon={IconNames.CARET_DOWN}
-                disabled={dimensions.length <= 1}
-              />
-            </Popover>
-          </FormGroup>
-        )}
+        {selectedDimensionSpec.index !== -1 &&
+          deepGet(spec, 'spec.dataSchema.metricsSpec') &&
+          !selectedTime && (
+            <FormGroup>
+              <Popover content={convertToMetricMenu}>
+                <Button
+                  icon={IconNames.EXCHANGE}
+                  text="Convert to metric"
+                  rightIcon={IconNames.CARET_DOWN}
+                  disabled={dimensions.length <= 1}
+                />
+              </Popover>
+            </FormGroup>
+          )}
       </FormEditor>
     );
   }
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
index ff3a54785ba..3bd7c36864a 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
@@ -137,11 +137,9 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
               <div
                 className="clickable"
                 onClick={() => {
-                  if (isTimestamp) return;
-
                   if (definedDimensions && dimensionSpec) {
                     onDimensionSelect(inflateDimensionSpec(dimensionSpec), dimensionSpecIndex);
-                  } else {
+                  } else if (!isTimestamp) {
                     onAutoDimensionSelect(columnName);
                   }
                 }}
diff --git a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
index 735a9ac2837..f083a1dbee8 100644
--- a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
+++ b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
@@ -22,6 +22,7 @@ import {
   ButtonGroup,
   Callout,
   FormGroup,
+  Icon,
   Intent,
   Menu,
   MenuDivider,
@@ -45,7 +46,13 @@ import { select, selectAll } from 'd3-selection';
 import type { JSX } from 'react';
 import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react';
 
-import { ClearableInput, LearnMore, Loader } from '../../../components';
+import {
+  ClearableInput,
+  ENABLE_DISABLE_OPTIONS_TEXT,
+  LearnMore,
+  Loader,
+  MenuBoolean,
+} from '../../../components';
 import { AsyncActionDialog } from '../../../dialogs';
 import type { Execution, ExternalConfig, IngestQueryPattern } from '../../../druid-models';
 import {
@@ -100,6 +107,8 @@ import { RollupAnalysisPane } from './rollup-analysis-pane/rollup-analysis-pane'
 
 import './schema-step.scss';
 
+const EXPERIMENTAL_ICON = <Icon icon={IconNames.WARNING_SIGN} title="Experimental" />;
+
 const queryRunner = new QueryRunner();
 
 function digestQueryString(queryString: string): {
@@ -248,6 +257,8 @@ interface EditorColumn {
 export interface SchemaStepProps {
   queryString: string;
   onQueryStringChange(queryString: string): void;
+  forceSegmentSortByTime: boolean;
+  changeForceSegmentSortByTime(forceSegmentSortByTime: boolean): void;
   enableAnalyze: boolean;
   goToQuery: () => void;
   onBack(): void;
@@ -259,6 +270,8 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
   const {
     queryString,
     onQueryStringChange,
+    forceSegmentSortByTime,
+    changeForceSegmentSortByTime,
     enableAnalyze,
     goToQuery,
     onBack,
@@ -640,6 +653,7 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
                       <MenuItem
                         icon={IconNames.CROSS}
                         text="Remove"
+                        shouldDismissPopover={false}
                         onClick={() =>
                           updatePattern({
                             ...ingestQueryPattern,
@@ -652,17 +666,13 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
                   <MenuItem icon={IconNames.PLUS} text="Add column clustering">
                     {filterMap(ingestQueryPattern.dimensions, (dimension, i) => {
                       const outputName = dimension.getOutputName();
-                      if (
-                        outputName === TIME_COLUMN ||
-                        ingestQueryPattern.clusteredBy.includes(i)
-                      ) {
-                        return;
-                      }
+                      if (ingestQueryPattern.clusteredBy.includes(i)) return;
 
                       return (
                         <MenuItem
                           key={i}
                           text={outputName}
+                          disabled={outputName === TIME_COLUMN && forceSegmentSortByTime}
                           onClick={() =>
                             updatePattern({
                               ...ingestQueryPattern,
@@ -674,6 +684,15 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
                       );
                     })}
                   </MenuItem>
+                  <MenuDivider />
+                  <MenuBoolean
+                    icon={IconNames.GEOTIME}
+                    text="Force segment sort by time"
+                    value={forceSegmentSortByTime}
+                    onValueChange={v => changeForceSegmentSortByTime(Boolean(v))}
+                    optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
+                    optionsLabelElement={{ false: EXPERIMENTAL_ICON }}
+                  />
                 </Menu>
               }
             >
diff --git a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
index 2e10734a170..a7d6adac71e 100644
--- a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
+++ b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
@@ -33,6 +33,7 @@ import {
   DEFAULT_SERVER_QUERY_CONTEXT,
   Execution,
   externalConfigToIngestQueryPattern,
+  getQueryContextKey,
   ingestQueryPatternToQuery,
 } from '../../druid-models';
 import type { Capabilities } from '../../helpers';
@@ -147,6 +148,17 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
         <SchemaStep
           queryString={content.queryString}
           onQueryStringChange={queryString => setContent({ ...content, queryString })}
+          forceSegmentSortByTime={getQueryContextKey(
+            'forceSegmentSortByTime',
+            content.queryContext || {},
+            serverQueryContext,
+          )}
+          changeForceSegmentSortByTime={forceSegmentSortByTime =>
+            setContent({
+              ...content,
+              queryContext: { ...content?.queryContext, forceSegmentSortByTime },
+            })
+          }
           enableAnalyze={false}
           goToQuery={() => goToQuery(content)}
           onBack={() => setContent(undefined)}
diff --git a/web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap b/web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap
index de5f9658d2c..d4326fdf6dc 100644
--- a/web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap
+++ b/web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap
@@ -80,10 +80,17 @@ exports[`MaxTasksButton matches snapshot 1`] = `
           active={false}
           disabled={false}
           icon="flow-branch"
-          label="max"
+          label="Max"
           multiline={false}
           popoverProps={{}}
           shouldDismissPopover={true}
+          submenuProps={
+            {
+              "style": {
+                "width": 300,
+              },
+            }
+          }
           text="Task assignment"
         >
           <Blueprint5.MenuItem
@@ -97,10 +104,10 @@ exports[`MaxTasksButton matches snapshot 1`] = `
             text={
               <React.Fragment>
                 <strong>
-                  max
+                  Max
                 </strong>
                 : 
-                Use as many tasks as possible, up to the maximum.
+                uses the maximum possible tasks up to the specified limit.
               </React.Fragment>
             }
           />
@@ -108,6 +115,13 @@ exports[`MaxTasksButton matches snapshot 1`] = `
             active={false}
             disabled={false}
             icon="blank"
+            labelElement={
+              <Blueprint5.Button
+                icon="help"
+                minimal={true}
+                onClick={[Function]}
+              />
+            }
             multiline={true}
             onClick={[Function]}
             popoverProps={{}}
@@ -115,10 +129,10 @@ exports[`MaxTasksButton matches snapshot 1`] = `
             text={
               <React.Fragment>
                 <strong>
-                  auto
+                  Auto
                 </strong>
                 : 
-                Use as few tasks as possible without exceeding 512 MiB or 10,000 files per task, unless exceeding these limits is necessary to stay within 'maxNumTasks'. When calculating the size of files, the weighted size is used, which considers the file format and compression format used if any. When file sizes cannot be determined through directory listing (for example: http), behaves the same as 'max'.
+                maximizes the number of tasks while staying within 512 MiB or 10,000 files per task, unless more tasks are needed to stay under the max task limit.
               </React.Fragment>
             }
           />
diff --git a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
index c84f9f00e37..21ba60f6b8d 100644
--- a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
+++ b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
@@ -19,20 +19,33 @@
 import type { ButtonProps } from '@blueprintjs/core';
 import { Button, Menu, MenuDivider, MenuItem, Popover, Position } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
-import type { JSX } from 'react';
+import type { JSX, ReactNode } from 'react';
 import React, { useState } from 'react';
 
 import { NumericInputDialog } from '../../../dialogs';
 import type { QueryContext, TaskAssignment } from '../../../druid-models';
 import { getQueryContextKey } from '../../../druid-models';
-import { deleteKeys, formatInteger, tickIcon } from '../../../utils';
+import { getLink } from '../../../links';
+import { capitalizeFirst, deleteKeys, formatInteger, tickIcon } from '../../../utils';
 
 const MAX_NUM_TASK_OPTIONS = [2, 3, 4, 5, 7, 9, 11, 17, 33, 65, 129];
 const TASK_ASSIGNMENT_OPTIONS: TaskAssignment[] = ['max', 'auto'];
 
 const TASK_ASSIGNMENT_DESCRIPTION: Record<string, string> = {
-  max: 'Use as many tasks as possible, up to the maximum.',
-  auto: `Use as few tasks as possible without exceeding 512 MiB or 10,000 files per task, unless exceeding these limits is necessary to stay within 'maxNumTasks'. When calculating the size of files, the weighted size is used, which considers the file format and compression format used if any. When file sizes cannot be determined through directory listing (for example: http), behaves the same as 'max'.`,
+  max: 'uses the maximum possible tasks up to the specified limit.',
+  auto: 'maximizes the number of tasks while staying within 512 MiB or 10,000 files per task, unless more tasks are needed to stay under the max task limit.',
+};
+
+const TASK_ASSIGNMENT_LABEL_ELEMENT: Record<string, ReactNode> = {
+  auto: (
+    <Button
+      icon={IconNames.HELP}
+      minimal
+      onClick={() =>
+        window.open(`${getLink('DOCS')}/multi-stage-query/reference#context-parameters`, '_blank')
+      }
+    />
+  ),
 };
 
 const DEFAULT_MAX_NUM_TASKS_LABEL_FN = (maxNum: number) => {
@@ -113,16 +126,22 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
               onClick={() => setCustomMaxNumTasksDialogOpen(true)}
             />
             <MenuDivider />
-            <MenuItem icon={IconNames.FLOW_BRANCH} text="Task assignment" label={taskAssigment}>
+            <MenuItem
+              icon={IconNames.FLOW_BRANCH}
+              text="Task assignment"
+              label={capitalizeFirst(taskAssigment)}
+              submenuProps={{ style: { width: 300 } }}
+            >
               {TASK_ASSIGNMENT_OPTIONS.map(t => (
                 <MenuItem
                   key={String(t)}
                   icon={tickIcon(t === taskAssigment)}
                   text={
                     <>
-                      <strong>{t}</strong>: {TASK_ASSIGNMENT_DESCRIPTION[t]}
+                      <strong>{capitalizeFirst(t)}</strong>: {TASK_ASSIGNMENT_DESCRIPTION[t]}
                     </>
                   }
+                  labelElement={TASK_ASSIGNMENT_LABEL_ELEMENT[t]}
                   shouldDismissPopover={false}
                   multiline
                   onClick={() => changeQueryContext({ ...queryContext, taskAssignment: t })}
diff --git a/web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap b/web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap
index 8c7f4ec7a70..51e5f34da2b 100644
--- a/web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap
+++ b/web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap
@@ -46,7 +46,7 @@ exports[`RunPanel matches snapshot on msq (auto) query 1`] = `
         <span
           class="bp5-button-text"
         >
-          Engine: sql-msq-task
+          Engine: SQL MSQ-task
         </span>
         <span
           aria-hidden="true"
@@ -150,7 +150,7 @@ exports[`RunPanel matches snapshot on native (auto) query 1`] = `
         <span
           class="bp5-button-text"
         >
-          Engine: auto (sql-native)
+          Engine: Auto (SQL native)
         </span>
         <span
           aria-hidden="true"
diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
index e329f9d383b..13901388b4e 100644
--- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx
+++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
@@ -33,7 +33,7 @@ import { IconNames } from '@blueprintjs/icons';
 import type { JSX } from 'react';
 import React, { useCallback, useMemo, useState } from 'react';
 
-import { MenuCheckbox, MenuTristate } from '../../../components';
+import { ENABLE_DISABLE_OPTIONS_TEXT, MenuBoolean, MenuCheckbox } from '../../../components';
 import { EditContextDialog, StringInputDialog } from '../../../dialogs';
 import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog';
 import type {
@@ -76,28 +76,52 @@ const NAMED_TIMEZONES: string[] = [
   'Australia/Sydney', // +11.0
 ];
 
+const ARRAY_INGEST_MODE_LABEL: Record<ArrayIngestMode, string> = {
+  array: 'Array',
+  mvd: 'MVD',
+};
 const ARRAY_INGEST_MODE_DESCRIPTION: Record<ArrayIngestMode, JSX.Element> = {
   array: (
     <>
-      array: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid{' '}
+      Array: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid{' '}
       <Tag minimal>ARRAY&lt;STRING&gt;</Tag>
     </>
   ),
   mvd: (
     <>
-      mvd: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid multi-value <Tag minimal>STRING</Tag>
+      MVD: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid multi-value <Tag minimal>STRING</Tag>
     </>
   ),
 };
 
-const DEFAULT_ENGINES_LABEL_FN = (engine: DruidEngine | undefined) => {
-  if (!engine) return { text: 'auto' };
-  return {
-    text: engine,
-    label: engine === 'sql-msq-task' ? 'multi-stage-query' : undefined,
-  };
+const SQL_JOIN_ALGORITHM_LABEL: Record<SqlJoinAlgorithm, string> = {
+  broadcast: 'Broadcast',
+  sortMerge: 'Sort merge',
 };
 
+const SELECT_DESTINATION_LABEL: Record<SelectDestination, string> = {
+  taskReport: 'Task report',
+  durableStorage: 'Durable storage',
+};
+
+const DEFAULT_ENGINES_LABEL_FN = (engine: DruidEngine | undefined) => {
+  switch (engine) {
+    case 'native':
+      return { text: 'Native' };
+
+    case 'sql-native':
+      return { text: 'SQL native' };
+
+    case 'sql-msq-task':
+      return { text: 'SQL MSQ-task', label: 'multi-stage-query' };
+
+    default:
+      return { text: 'Auto' };
+  }
+};
+
+const EXPERIMENTAL_ICON = <Icon icon={IconNames.WARNING_SIGN} title="Experimental" />;
+
 export interface RunPanelProps
   extends Pick<MaxTasksButtonProps, 'maxTasksLabelFn' | 'fullClusterCapacityLabelFn'> {
   query: WorkbenchQuery;
@@ -169,6 +193,11 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
     queryContext,
     defaultQueryContext,
   );
+  const forceSegmentSortByTime = getQueryContextKey(
+    'forceSegmentSortByTime',
+    queryContext,
+    defaultQueryContext,
+  );
   const finalizeAggregations = queryContext.finalizeAggregations;
   const waitUntilSegmentsLoad = queryContext.waitUntilSegmentsLoad;
   const groupByEnableMultiValueUnnesting = queryContext.groupByEnableMultiValueUnnesting;
@@ -313,13 +342,13 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                 )}
                 <MenuItem
                   icon={IconNames.PROPERTIES}
-                  text="Edit context"
+                  text="Edit query context..."
                   onClick={() => setEditContextDialogOpen(true)}
                   label={pluralIfNeeded(numContextKeys, 'key')}
                 />
                 <MenuItem
                   icon={IconNames.HELP}
-                  text="Define parameters"
+                  text="Define parameters..."
                   onClick={() => setEditParametersDialogOpen(true)}
                   label={queryParameters ? pluralIfNeeded(queryParameters.length, 'parameter') : ''}
                 />
@@ -365,37 +394,52 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                 {effectiveEngine === 'sql-msq-task' ? (
                   <>
                     <MenuItem icon={IconNames.BRING_DATA} text="INSERT / REPLACE specific context">
-                      <MenuCheckbox
-                        checked={useConcurrentLocks}
-                        text="Use concurrent locks"
-                        onChange={() =>
+                      <MenuBoolean
+                        text="Force segment sort by time"
+                        value={forceSegmentSortByTime}
+                        onValueChange={forceSegmentSortByTime =>
                           changeQueryContext({
                             ...queryContext,
-                            useConcurrentLocks: !useConcurrentLocks,
+                            forceSegmentSortByTime,
                           })
                         }
+                        optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
+                        optionsLabelElement={{ false: EXPERIMENTAL_ICON }}
                       />
-                      <MenuTristate
-                        icon={IconNames.DISABLE}
+                      <MenuBoolean
+                        text="Use concurrent locks"
+                        value={useConcurrentLocks}
+                        onValueChange={useConcurrentLocks =>
+                          changeQueryContext({
+                            ...queryContext,
+                            useConcurrentLocks,
+                          })
+                        }
+                        optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
+                        optionsLabelElement={{ true: EXPERIMENTAL_ICON }}
+                      />
+                      <MenuBoolean
                         text="Fail on empty insert"
                         value={failOnEmptyInsert}
+                        showUndefined
                         undefinedEffectiveValue={false}
                         onValueChange={failOnEmptyInsert =>
                           changeQueryContext({ ...queryContext, failOnEmptyInsert })
                         }
+                        optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
                       />
-                      <MenuTristate
-                        icon={IconNames.STOPWATCH}
+                      <MenuBoolean
                         text="Wait until segments have loaded"
                         value={waitUntilSegmentsLoad}
+                        showUndefined
                         undefinedEffectiveValue={ingestMode}
                         onValueChange={waitUntilSegmentsLoad =>
                           changeQueryContext({ ...queryContext, waitUntilSegmentsLoad })
                         }
+                        optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
                       />
                       <MenuItem
-                        icon={IconNames.TH_DERIVED}
-                        text="Edit index spec"
+                        text="Edit index spec..."
                         label={summarizeIndexSpec(indexSpec)}
                         shouldDismissPopover={false}
                         onClick={() => {
@@ -420,34 +464,41 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                         />
                       ))}
                     </MenuItem>
-                    <MenuTristate
+                    <MenuBoolean
                       icon={IconNames.TRANSLATE}
                       text="Finalize aggregations"
                       value={finalizeAggregations}
+                      showUndefined
                       undefinedEffectiveValue={!ingestMode}
                       onValueChange={finalizeAggregations =>
                         changeQueryContext({ ...queryContext, finalizeAggregations })
                       }
+                      optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
                     />
-                    <MenuTristate
+                    <MenuBoolean
                       icon={IconNames.FORK}
-                      text="Enable GroupBy multi-value unnesting"
+                      text="GROUP BY multi-value unnesting"
                       value={groupByEnableMultiValueUnnesting}
+                      showUndefined
                       undefinedEffectiveValue={!ingestMode}
                       onValueChange={groupByEnableMultiValueUnnesting =>
                         changeQueryContext({ ...queryContext, groupByEnableMultiValueUnnesting })
                       }
+                      optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
                     />
                     <MenuItem
                       icon={IconNames.INNER_JOIN}
                       text="Join algorithm"
-                      label={sqlJoinAlgorithm}
+                      label={
+                        SQL_JOIN_ALGORITHM_LABEL[sqlJoinAlgorithm as SqlJoinAlgorithm] ??
+                        sqlJoinAlgorithm
+                      }
                     >
                       {(['broadcast', 'sortMerge'] as SqlJoinAlgorithm[]).map(o => (
                         <MenuItem
                           key={o}
                           icon={tickIcon(sqlJoinAlgorithm === o)}
-                          text={o}
+                          text={SQL_JOIN_ALGORITHM_LABEL[o]}
                           shouldDismissPopover={false}
                           onClick={() =>
                             changeQueryContext({ ...queryContext, sqlJoinAlgorithm: o })
@@ -458,14 +509,17 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                     <MenuItem
                       icon={IconNames.MANUALLY_ENTERED_DATA}
                       text="SELECT destination"
-                      label={selectDestination}
+                      label={
+                        SELECT_DESTINATION_LABEL[selectDestination as SelectDestination] ??
+                        selectDestination
+                      }
                       intent={intent}
                     >
                       {(['taskReport', 'durableStorage'] as SelectDestination[]).map(o => (
                         <MenuItem
                           key={o}
                           icon={tickIcon(selectDestination === o)}
-                          text={o}
+                          text={SELECT_DESTINATION_LABEL[o]}
                           shouldDismissPopover={false}
                           onClick={() =>
                             changeQueryContext({ ...queryContext, selectDestination: o })
@@ -486,52 +540,60 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                         }}
                       />
                     </MenuItem>
-                    <MenuCheckbox
-                      checked={durableShuffleStorage}
+                    <MenuBoolean
+                      icon={IconNames.CLOUD_TICK}
                       text="Durable shuffle storage"
-                      onChange={() =>
+                      value={durableShuffleStorage}
+                      onValueChange={durableShuffleStorage =>
                         changeQueryContext({
                           ...queryContext,
-                          durableShuffleStorage: !durableShuffleStorage,
+                          durableShuffleStorage,
                         })
                       }
+                      optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
                     />
                   </>
                 ) : (
                   <>
-                    <MenuCheckbox
-                      checked={useCache}
+                    <MenuBoolean
+                      icon={IconNames.DATA_CONNECTION}
                       text="Use cache"
-                      onChange={() =>
+                      value={useCache}
+                      onValueChange={useCache =>
                         changeQueryContext({
                           ...queryContext,
-                          useCache: !useCache,
-                          populateCache: !useCache,
+                          useCache,
+                          populateCache: useCache,
                         })
                       }
+                      optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
                     />
-                    <MenuCheckbox
-                      checked={useApproximateTopN}
-                      text="Use approximate TopN"
-                      onChange={() =>
+                    <MenuBoolean
+                      icon={IconNames.HORIZONTAL_BAR_CHART_DESC}
+                      text="Approximate TopN"
+                      value={useApproximateTopN}
+                      onValueChange={useApproximateTopN =>
                         changeQueryContext({
                           ...queryContext,
-                          useApproximateTopN: !useApproximateTopN,
+                          useApproximateTopN,
                         })
                       }
+                      optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
                     />
                   </>
                 )}
                 {effectiveEngine !== 'native' && (
-                  <MenuCheckbox
-                    checked={useApproximateCountDistinct}
-                    text="Use approximate COUNT(DISTINCT)"
-                    onChange={() =>
+                  <MenuBoolean
+                    icon={IconNames.ROCKET_SLANT}
+                    text="Approximate COUNT(DISTINCT)"
+                    value={useApproximateCountDistinct}
+                    onValueChange={useApproximateCountDistinct =>
                       changeQueryContext({
                         ...queryContext,
-                        useApproximateCountDistinct: !useApproximateCountDistinct,
+                        useApproximateCountDistinct,
                       })
                     }
+                    optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
                   />
                 )}
                 {effectiveEngine === 'sql-native' && (
@@ -603,7 +665,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
               }
             >
               <Button
-                text={`Array ingest mode: ${arrayIngestMode ?? '(server default)'}`}
+                text={`Array ingest mode: ${
+                  arrayIngestMode ? ARRAY_INGEST_MODE_LABEL[arrayIngestMode] : '(server default)'
+                }`}
                 rightIcon={IconNames.CARET_DOWN}
               />
             </Popover>

From d5b64ba2e3a4848f4d62157d9d50fec4953563fc Mon Sep 17 00:00:00 2001
From: Kashif Faraz <kashif.faraz@gmail.com>
Date: Fri, 30 Aug 2024 00:02:49 -0700
Subject: [PATCH 68/99] Improve exception handling in extension druid-pac4j
 (#16979)

Changes:
- Simplify exception handling in `CryptoService` by just catching a `Exception`
- Throw a `DruidException` as the exception is user facing
- Log the exception for easier debugging
- Add a test to verify thrown exception
---
 .../security/pac4j/Pac4jSessionStoreTest.java | 42 +++++++++++++++++--
 .../apache/druid/crypto/CryptoService.java    | 20 ++++-----
 2 files changed, 49 insertions(+), 13 deletions(-)

diff --git a/extensions-core/druid-pac4j/src/test/java/org/apache/druid/security/pac4j/Pac4jSessionStoreTest.java b/extensions-core/druid-pac4j/src/test/java/org/apache/druid/security/pac4j/Pac4jSessionStoreTest.java
index 772bef7ef6c..e2f46a4062e 100644
--- a/extensions-core/druid-pac4j/src/test/java/org/apache/druid/security/pac4j/Pac4jSessionStoreTest.java
+++ b/extensions-core/druid-pac4j/src/test/java/org/apache/druid/security/pac4j/Pac4jSessionStoreTest.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.security.pac4j;
 
+import org.apache.druid.error.DruidException;
 import org.easymock.Capture;
 import org.easymock.EasyMock;
 import org.junit.Assert;
@@ -41,7 +42,7 @@ public class Pac4jSessionStoreTest
   @Test
   public void testSetAndGet()
   {
-    Pac4jSessionStore<WebContext> sessionStore = new Pac4jSessionStore(COOKIE_PASSPHRASE);
+    Pac4jSessionStore<WebContext> sessionStore = new Pac4jSessionStore<>(COOKIE_PASSPHRASE);
 
     WebContext webContext1 = EasyMock.mock(WebContext.class);
     EasyMock.expect(webContext1.getScheme()).andReturn("https");
@@ -68,7 +69,7 @@ public class Pac4jSessionStoreTest
   @Test
   public void testSetAndGetClearUserProfile()
   {
-    Pac4jSessionStore<WebContext> sessionStore = new Pac4jSessionStore(COOKIE_PASSPHRASE);
+    Pac4jSessionStore<WebContext> sessionStore = new Pac4jSessionStore<>(COOKIE_PASSPHRASE);
 
     WebContext webContext1 = EasyMock.mock(WebContext.class);
     EasyMock.expect(webContext1.getScheme()).andReturn("https");
@@ -78,6 +79,7 @@ public class Pac4jSessionStoreTest
     EasyMock.replay(webContext1);
 
     CommonProfile profile = new CommonProfile();
+    profile.setId("profile1");
     profile.addAttribute(CommonProfileDefinition.DISPLAY_NAME, "name");
     sessionStore.set(webContext1, "pac4jUserProfiles", profile);
 
@@ -99,7 +101,7 @@ public class Pac4jSessionStoreTest
   @Test
   public void testSetAndGetClearUserMultipleProfile()
   {
-    Pac4jSessionStore<WebContext> sessionStore = new Pac4jSessionStore(COOKIE_PASSPHRASE);
+    Pac4jSessionStore<WebContext> sessionStore = new Pac4jSessionStore<>(COOKIE_PASSPHRASE);
 
     WebContext webContext1 = EasyMock.mock(WebContext.class);
     EasyMock.expect(webContext1.getScheme()).andReturn("https");
@@ -109,8 +111,10 @@ public class Pac4jSessionStoreTest
     EasyMock.replay(webContext1);
 
     CommonProfile profile1 = new CommonProfile();
+    profile1.setId("profile1");
     profile1.addAttribute(CommonProfileDefinition.DISPLAY_NAME, "name1");
     CommonProfile profile2 = new CommonProfile();
+    profile2.setId("profile2");
     profile2.addAttribute(CommonProfileDefinition.DISPLAY_NAME, "name2");
     Map<String, CommonProfile> profiles = new HashMap<>();
     profiles.put("profile1", profile1);
@@ -131,4 +135,36 @@ public class Pac4jSessionStoreTest
     Assert.assertTrue(Objects.requireNonNull(value).isPresent());
     Assert.assertEquals(2, ((Map<String, CommonProfile>) value.get()).size());
   }
+
+  @Test
+  public void testGetWithWrongPassphraseThrowsDruidException()
+  {
+    final WebContext webContext = EasyMock.mock(WebContext.class);
+    EasyMock.expect(webContext.getScheme()).andReturn("https");
+
+    final Capture<Cookie> cookieCapture = EasyMock.newCapture();
+    EasyMock.expect(webContext.getRequestCookies())
+            .andAnswer(() -> Collections.singleton(cookieCapture.getValue()));
+
+    webContext.addResponseCookie(EasyMock.capture(cookieCapture));
+    EasyMock.expectLastCall().once();
+    EasyMock.replay(webContext);
+
+    // Create a cookie with an invalid passphrase
+    new Pac4jSessionStore<>("invalid-passphrase").set(webContext, "key", "value");
+
+    // Verify that trying to decrypt the invalid cookie throws an exception
+    final Pac4jSessionStore<WebContext> sessionStore = new Pac4jSessionStore<>(COOKIE_PASSPHRASE);
+    DruidException exception = Assert.assertThrows(
+        DruidException.class,
+        () -> sessionStore.get(webContext, "key")
+    );
+    Assert.assertEquals(
+        "Decryption failed. Check service logs.",
+        exception.getMessage()
+    );
+    Assert.assertNull(exception.getCause());
+
+    EasyMock.verify(webContext);
+  }
 }
diff --git a/processing/src/main/java/org/apache/druid/crypto/CryptoService.java b/processing/src/main/java/org/apache/druid/crypto/CryptoService.java
index cd4b3279636..92d4f6602c4 100644
--- a/processing/src/main/java/org/apache/druid/crypto/CryptoService.java
+++ b/processing/src/main/java/org/apache/druid/crypto/CryptoService.java
@@ -20,25 +20,21 @@
 package org.apache.druid.crypto;
 
 import com.google.common.base.Preconditions;
+import org.apache.druid.error.InternalServerError;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.logger.Logger;
 
 import javax.annotation.Nullable;
-import javax.crypto.BadPaddingException;
 import javax.crypto.Cipher;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
 import javax.crypto.SecretKey;
 import javax.crypto.SecretKeyFactory;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.PBEKeySpec;
 import javax.crypto.spec.SecretKeySpec;
 import java.nio.ByteBuffer;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.spec.InvalidKeySpecException;
-import java.security.spec.InvalidParameterSpecException;
 import java.security.spec.KeySpec;
 
 /**
@@ -50,6 +46,8 @@ import java.security.spec.KeySpec;
  */
 public class CryptoService
 {
+  private static final Logger log = new Logger(CryptoService.class);
+
   // Based on Javadocs on SecureRandom, It is threadsafe as well.
   private static final SecureRandom SECURE_RANDOM_INSTANCE = new SecureRandom();
 
@@ -125,8 +123,9 @@ public class CryptoService
           ecipher.doFinal(plain)
       ).toByteAray();
     }
-    catch (InvalidKeySpecException | NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidParameterSpecException | IllegalBlockSizeException | BadPaddingException ex) {
-      throw new RuntimeException(ex);
+    catch (Exception ex) {
+      log.noStackTrace().warn(ex, "Encryption failed");
+      throw InternalServerError.exception("Encryption failed. Check service logs.");
     }
   }
 
@@ -145,8 +144,9 @@ public class CryptoService
       dcipher.init(Cipher.DECRYPT_MODE, secret, new IvParameterSpec(encryptedData.getIv()));
       return dcipher.doFinal(encryptedData.getCipher());
     }
-    catch (InvalidKeySpecException | NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException | InvalidKeyException | IllegalBlockSizeException | BadPaddingException ex) {
-      throw new RuntimeException(ex);
+    catch (Exception ex) {
+      log.noStackTrace().warn(ex, "Decryption failed");
+      throw InternalServerError.exception("Decryption failed. Check service logs.");
     }
   }
 

From caf8ce3e0b052d67c56620de459f1a27ba6f874f Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Fri, 30 Aug 2024 20:02:30 -0700
Subject: [PATCH 69/99] MSQ: Add CPU and thread usage counters. (#16914)

* MSQ: Add CPU and thread usage counters.

The main change adds "cpu" and "wall" counters. The "cpu" counter measures
CPU time (using JvmUtils.getCurrentThreadCpuTime) taken up by processors
in processing threads. The "wall" counter measures the amount of wall time
taken up by processors in those same processing threads. Both counters are
broken down by type of processor.

This patch also includes changes to support adding new counters. Due to an
oversight in the original design, older deserializers are not forwards-compatible;
they throw errors when encountering an unknown counter type. To manage this,
the following changes are made:

1) The defaultImpl NilQueryCounterSnapshot is added to QueryCounterSnapshot's
   deserialization configuration. This means that any unrecognized counter types
   will be read as "nil" by deserializers. Going forward, once all servers are
   on the latest code, this is enough to enable easily adding new counters.

2) A new context parameter "includeAllCounters" is added, which defaults to "false".
   When this parameter is set "false", only legacy counters are included. When set
   to "true", all counters are included. This is currently undocumented. In a future
   version, we should set the default to "true", and at that time, include a release
   note that people updating from versions prior to Druid 31 should set this to
   "false" until their upgrade is complete.

* Style, coverage.

* Fix.
---
 .../druid/msq/counters/CounterNames.java      |   9 ++
 .../druid/msq/counters/CounterSnapshots.java  |   1 +
 .../druid/msq/counters/CounterTracker.java    |  63 +++++++-
 .../apache/druid/msq/counters/CpuCounter.java | 148 ++++++++++++++++++
 .../druid/msq/counters/CpuCounters.java       | 106 +++++++++++++
 .../CpuTimeAccumulatingFrameProcessor.java    |  92 +++++++++++
 .../CpuTimeAccumulatingProcessorManager.java  |  74 +++++++++
 .../msq/counters/NilQueryCounterSnapshot.java |  51 ++++++
 .../msq/counters/QueryCounterSnapshot.java    |   8 +-
 .../apache/druid/msq/exec/ControllerImpl.java |   1 +
 .../apache/druid/msq/exec/RunWorkOrder.java   |  44 ++++--
 .../org/apache/druid/msq/exec/WorkerImpl.java |   3 +-
 .../druid/msq/guice/MSQIndexingModule.java    |   6 +
 .../indexing/IndexerControllerContext.java    |   4 +-
 .../msq/indexing/IndexerWorkerContext.java    |   6 +-
 .../druid/msq/indexing/InputChannelsImpl.java |  26 ++-
 .../msq/util/MultiStageQueryContext.java      |  23 +++
 .../counters/CountersSnapshotTreeTest.java    |  96 ++++++++++++
 .../druid/msq/counters/CpuCountersTest.java   |  86 ++++++++++
 .../processor/FrameProcessorDecorator.java    |  37 +++++
 .../druid/frame/processor/SuperSorter.java    |   5 +-
 .../frame/processor/SuperSorterTest.java      |   3 +
 22 files changed, 871 insertions(+), 21 deletions(-)
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuCounter.java
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuCounters.java
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuTimeAccumulatingFrameProcessor.java
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuTimeAccumulatingProcessorManager.java
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/NilQueryCounterSnapshot.java
 create mode 100644 extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CpuCountersTest.java
 create mode 100644 processing/src/main/java/org/apache/druid/frame/processor/FrameProcessorDecorator.java

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterNames.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterNames.java
index 350dc958e7b..19d5da36efb 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterNames.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterNames.java
@@ -34,6 +34,7 @@ public class CounterNames
   private static final String INPUT = "input";
   private static final String OUTPUT = "output";
   private static final String SHUFFLE = "shuffle";
+  private static final String CPU = "cpu";
   private static final String SORT_PROGRESS = "sortProgress";
   private static final String SEGMENT_GENERATION_PROGRESS = "segmentGenerationProgress";
   private static final String WARNINGS = "warnings";
@@ -68,6 +69,14 @@ public class CounterNames
     return SHUFFLE;
   }
 
+  /**
+   * Standard name for CPU counters created by {@link CounterTracker#cpu}.
+   */
+  public static String cpu()
+  {
+    return CPU;
+  }
+
   /**
    * Standard name for a sort progress counter created by {@link CounterTracker#sortProgress()}.
    */
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshots.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshots.java
index d75eb1ce8cb..b1595e9a0ae 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshots.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshots.java
@@ -27,6 +27,7 @@ import java.util.Objects;
 
 /**
  * Named counter snapshots. Immutable. Often part of a {@link CounterSnapshotsTree}.
+ * Created by {@link CounterTracker#snapshot()}.
  */
 public class CounterSnapshots
 {
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterTracker.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterTracker.java
index c73ead63c11..599a4df92b0 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterTracker.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterTracker.java
@@ -19,7 +19,12 @@
 
 package org.apache.druid.msq.counters;
 
+import org.apache.druid.frame.processor.FrameProcessor;
 import org.apache.druid.frame.processor.SuperSorterProgressTracker;
+import org.apache.druid.frame.processor.manager.ProcessorManager;
+import org.apache.druid.msq.util.MultiStageQueryContext;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.utils.JvmUtils;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -37,11 +42,55 @@ public class CounterTracker
 {
   private final ConcurrentHashMap<String, QueryCounter> countersMap = new ConcurrentHashMap<>();
 
+  /**
+   * See {@link MultiStageQueryContext#getIncludeAllCounters(QueryContext)}.
+   */
+  private final boolean includeAllCounters;
+
+  public CounterTracker(boolean includeAllCounters)
+  {
+    this.includeAllCounters = includeAllCounters;
+  }
+
   public ChannelCounters channel(final String name)
   {
     return counter(name, ChannelCounters::new);
   }
 
+  /**
+   * Returns a {@link CpuCounter} that can be used to accumulate CPU time under a particular label.
+   */
+  public CpuCounter cpu(final String name)
+  {
+    return counter(CounterNames.cpu(), CpuCounters::new).forName(name);
+  }
+
+  /**
+   * Decorates a {@link FrameProcessor} such that it accumulates CPU time under a particular label.
+   */
+  public <T> FrameProcessor<T> trackCpu(final FrameProcessor<T> processor, final String name)
+  {
+    if (JvmUtils.isThreadCpuTimeEnabled()) {
+      final CpuCounter counter = counter(CounterNames.cpu(), CpuCounters::new).forName(name);
+      return new CpuTimeAccumulatingFrameProcessor<>(processor, counter);
+    } else {
+      return processor;
+    }
+  }
+
+  /**
+   * Decorates a {@link ProcessorManager} such that it accumulates CPU time under a particular label.
+   */
+  public <T, R> ProcessorManager<T, R> trackCpu(final ProcessorManager<T, R> processorManager, final String name)
+  {
+    if (JvmUtils.isThreadCpuTimeEnabled()) {
+      final CpuCounter counter = counter(CounterNames.cpu(), CpuCounters::new).forName(name);
+      return new CpuTimeAccumulatingProcessorManager<>(processorManager, counter);
+    } else {
+      return processorManager;
+    }
+  }
+
   public SuperSorterProgressTracker sortProgress()
   {
     return counter(CounterNames.sortProgress(), SuperSorterProgressTrackerCounter::new).tracker();
@@ -69,11 +118,23 @@ public class CounterTracker
 
     for (final Map.Entry<String, QueryCounter> entry : countersMap.entrySet()) {
       final QueryCounterSnapshot counterSnapshot = entry.getValue().snapshot();
-      if (counterSnapshot != null) {
+      if (counterSnapshot != null && (includeAllCounters || isLegacyCounter(counterSnapshot))) {
         m.put(entry.getKey(), counterSnapshot);
       }
     }
 
     return new CounterSnapshots(m);
   }
+
+  /**
+   * Returns whether a counter is a "legacy counter" that can be snapshotted regardless of the value of
+   * {@link MultiStageQueryContext#getIncludeAllCounters(QueryContext)}.
+   */
+  private static boolean isLegacyCounter(final QueryCounterSnapshot counterSnapshot)
+  {
+    return counterSnapshot instanceof ChannelCounters.Snapshot
+        || counterSnapshot instanceof SuperSorterProgressTrackerCounter.Snapshot
+        || counterSnapshot instanceof WarningCounters.Snapshot
+        || counterSnapshot instanceof SegmentGenerationProgressCounter.Snapshot;
+  }
 }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuCounter.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuCounter.java
new file mode 100644
index 00000000000..747d0e5aa3b
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuCounter.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.counters;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.druid.utils.JvmUtils;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class CpuCounter implements QueryCounter
+{
+  private final AtomicLong cpuTime = new AtomicLong();
+  private final AtomicLong wallTime = new AtomicLong();
+
+  public void accumulate(final long cpu, final long wall)
+  {
+    cpuTime.addAndGet(cpu);
+    wallTime.addAndGet(wall);
+  }
+
+  public <E extends Throwable> void run(final Doer<E> doer) throws E
+  {
+    final long startCpu = JvmUtils.getCurrentThreadCpuTime();
+    final long startWall = System.nanoTime();
+
+    try {
+      doer.run();
+    }
+    finally {
+      accumulate(
+          JvmUtils.getCurrentThreadCpuTime() - startCpu,
+          System.nanoTime() - startWall
+      );
+    }
+  }
+
+  public <T, E extends Throwable> T run(final Returner<T, E> returner) throws E
+  {
+    final long startCpu = JvmUtils.getCurrentThreadCpuTime();
+    final long startWall = System.nanoTime();
+
+    try {
+      return returner.run();
+    }
+    finally {
+      accumulate(
+          JvmUtils.getCurrentThreadCpuTime() - startCpu,
+          System.nanoTime() - startWall
+      );
+    }
+  }
+
+  @Override
+  public Snapshot snapshot()
+  {
+    return new Snapshot(cpuTime.get(), wallTime.get());
+  }
+
+  @JsonTypeName("cpu")
+  public static class Snapshot implements QueryCounterSnapshot
+  {
+    private final long cpuTime;
+    private final long wallTime;
+
+    @JsonCreator
+    public Snapshot(
+        @JsonProperty("cpu") long cpuTime,
+        @JsonProperty("wall") long wallTime
+    )
+    {
+      this.cpuTime = cpuTime;
+      this.wallTime = wallTime;
+    }
+
+    @JsonProperty("cpu")
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+    public long getCpuTime()
+    {
+      return cpuTime;
+    }
+
+    @JsonProperty("wall")
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+    public long getWallTime()
+    {
+      return wallTime;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      Snapshot snapshot = (Snapshot) o;
+      return cpuTime == snapshot.cpuTime && wallTime == snapshot.wallTime;
+    }
+
+    @Override
+    public int hashCode()
+    {
+      return Objects.hash(cpuTime, wallTime);
+    }
+
+    @Override
+    public String toString()
+    {
+      return "CpuCounter.Snapshot{" +
+             "cpuTime=" + cpuTime +
+             ", wallTime=" + wallTime +
+             '}';
+    }
+  }
+
+  public interface Doer<E extends Throwable>
+  {
+    void run() throws E;
+  }
+
+  public interface Returner<T, E extends Throwable>
+  {
+    T run() throws E;
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuCounters.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuCounters.java
new file mode 100644
index 00000000000..8ab79b302c6
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuCounters.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.counters;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.base.Preconditions;
+
+import javax.annotation.Nullable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class CpuCounters implements QueryCounter
+{
+  public static final String LABEL_MAIN = "main";
+  public static final String LABEL_KEY_STATISTICS = "collectKeyStatistics";
+  public static final String LABEL_MERGE_INPUT = "mergeInput";
+  public static final String LABEL_HASH_PARTITION = "hashPartitionOutput";
+  public static final String LABEL_MIX = "mixOutput";
+  public static final String LABEL_SORT = "sortOutput";
+
+  private final ConcurrentHashMap<String, CpuCounter> counters = new ConcurrentHashMap<>();
+
+  public CpuCounter forName(final String name)
+  {
+    return counters.computeIfAbsent(name, k -> new CpuCounter());
+  }
+
+  @Nullable
+  @Override
+  public CpuCounters.Snapshot snapshot()
+  {
+    final Map<String, CpuCounter.Snapshot> snapshotMap = new HashMap<>();
+    for (Map.Entry<String, CpuCounter> entry : counters.entrySet()) {
+      snapshotMap.put(entry.getKey(), entry.getValue().snapshot());
+    }
+    return new Snapshot(snapshotMap);
+  }
+
+  @JsonTypeName("cpus")
+  public static class Snapshot implements QueryCounterSnapshot
+  {
+    // String keys, not enum, so deserialization is forwards-compatible
+    private final Map<String, CpuCounter.Snapshot> map;
+
+    @JsonCreator
+    public Snapshot(Map<String, CpuCounter.Snapshot> map)
+    {
+      this.map = Preconditions.checkNotNull(map, "map");
+    }
+
+    @JsonValue
+    public Map<String, CpuCounter.Snapshot> getCountersMap()
+    {
+      return map;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      Snapshot snapshot = (Snapshot) o;
+      return Objects.equals(map, snapshot.map);
+    }
+
+    @Override
+    public int hashCode()
+    {
+      return Objects.hash(map);
+    }
+
+    @Override
+    public String toString()
+    {
+      return "CpuCounters.Snapshot{" +
+             "map=" + map +
+             '}';
+    }
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuTimeAccumulatingFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuTimeAccumulatingFrameProcessor.java
new file mode 100644
index 00000000000..5cb6866a5c4
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuTimeAccumulatingFrameProcessor.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.counters;
+
+import it.unimi.dsi.fastutil.ints.IntSet;
+import org.apache.druid.frame.channel.ReadableFrameChannel;
+import org.apache.druid.frame.channel.WritableFrameChannel;
+import org.apache.druid.frame.processor.FrameProcessor;
+import org.apache.druid.frame.processor.ReturnOrAwait;
+import org.apache.druid.utils.JvmUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Wrapper around {@link FrameProcessor} that accumulates time taken into a {@link CpuCounter}.
+ */
+public class CpuTimeAccumulatingFrameProcessor<T> implements FrameProcessor<T>
+{
+  private final FrameProcessor<T> delegate;
+  private final CpuCounter counter;
+
+  public CpuTimeAccumulatingFrameProcessor(final FrameProcessor<T> delegate, final CpuCounter counter)
+  {
+    this.delegate = delegate;
+    this.counter = counter;
+  }
+
+  @Override
+  public List<ReadableFrameChannel> inputChannels()
+  {
+    return delegate.inputChannels();
+  }
+
+  @Override
+  public List<WritableFrameChannel> outputChannels()
+  {
+    return delegate.outputChannels();
+  }
+
+  @Override
+  public ReturnOrAwait<T> runIncrementally(IntSet readableInputs) throws InterruptedException, IOException
+  {
+    // Can't use counter.run, because it turns "throws InterruptedException, IOException" into "throws Exception".
+    final long startCpu = JvmUtils.getCurrentThreadCpuTime();
+    final long startWall = System.nanoTime();
+
+    try {
+      return delegate.runIncrementally(readableInputs);
+    }
+    finally {
+      counter.accumulate(
+          JvmUtils.getCurrentThreadCpuTime() - startCpu,
+          System.nanoTime() - startWall
+      );
+    }
+  }
+
+  @Override
+  public void cleanup() throws IOException
+  {
+    final long startCpu = JvmUtils.getCurrentThreadCpuTime();
+    final long startWall = System.nanoTime();
+
+    try {
+      delegate.cleanup();
+    }
+    finally {
+      counter.accumulate(
+          JvmUtils.getCurrentThreadCpuTime() - startCpu,
+          System.nanoTime() - startWall
+      );
+    }
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuTimeAccumulatingProcessorManager.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuTimeAccumulatingProcessorManager.java
new file mode 100644
index 00000000000..6e04e1d3c18
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CpuTimeAccumulatingProcessorManager.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.counters;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.druid.common.guava.FutureUtils;
+import org.apache.druid.frame.processor.manager.ProcessorAndCallback;
+import org.apache.druid.frame.processor.manager.ProcessorManager;
+
+import java.util.Optional;
+
+/**
+ * Wrapper around {@link ProcessorManager} that accumulates time taken into a {@link CpuCounter}.
+ */
+public class CpuTimeAccumulatingProcessorManager<T, R> implements ProcessorManager<T, R>
+{
+  private final ProcessorManager<T, R> delegate;
+  private final CpuCounter counter;
+
+  public CpuTimeAccumulatingProcessorManager(ProcessorManager<T, R> delegate, CpuCounter counter)
+  {
+    this.delegate = delegate;
+    this.counter = counter;
+  }
+
+  @Override
+  public ListenableFuture<Optional<ProcessorAndCallback<T>>> next()
+  {
+    // Measure time taken by delegate.next()
+    final ListenableFuture<Optional<ProcessorAndCallback<T>>> delegateNext = counter.run(delegate::next);
+
+    return FutureUtils.transform(
+        delegateNext,
+
+        // Don't bother measuring time taken by opt.map, it's very quick.
+        opt -> opt.map(
+            pac -> new ProcessorAndCallback<>(
+                new CpuTimeAccumulatingFrameProcessor<>(pac.processor(), counter),
+                // Do measure time taken by onComplete(t), though.
+                t -> counter.run(() -> pac.onComplete(t))
+            )
+        )
+    );
+  }
+
+  @Override
+  public R result()
+  {
+    return counter.run(delegate::result);
+  }
+
+  @Override
+  public void close()
+  {
+    counter.run(delegate::close);
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/NilQueryCounterSnapshot.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/NilQueryCounterSnapshot.java
new file mode 100644
index 00000000000..eca3f6d6228
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/NilQueryCounterSnapshot.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.counters;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+/**
+ * Represents an unknown counter type. This is the "defaultType" for {@link QueryCounterSnapshot}, so it is
+ * substituted at deserialization time if the type is unknown. This can happen when running mixed versions, where some
+ * servers support a newer counter type and some don't.
+ */
+@JsonTypeName("nil")
+public class NilQueryCounterSnapshot implements QueryCounterSnapshot
+{
+  private NilQueryCounterSnapshot()
+  {
+    // Singleton
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return 0;
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    return o != null && getClass() == o.getClass();
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/QueryCounterSnapshot.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/QueryCounterSnapshot.java
index c065f7f8252..0fe03088ae7 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/QueryCounterSnapshot.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/QueryCounterSnapshot.java
@@ -24,8 +24,14 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
 /**
  * Marker interface for the results of {@link QueryCounter#snapshot()}. No methods, because the only purpose of these
  * snapshots is to pass things along from worker -> controller -> report.
+ *
+ * To support easy adding of new counters, implementations must use forward-compatible deserialization setups.
+ * In particular, implementations should avoid using enums where new values may be added in the future.
+ *
+ * The default impl is {@link NilQueryCounterSnapshot}. This means that readers will see {@link NilQueryCounterSnapshot}
+ * if they don't understand the particular counter type in play.
  */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = NilQueryCounterSnapshot.class)
 public interface QueryCounterSnapshot
 {
 }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
index 46a24611351..77a0b7d48d6 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java
@@ -2705,6 +2705,7 @@ public class ControllerImpl implements Controller
             () -> ArenaMemoryAllocator.createOnHeap(5_000_000),
             resultReaderExec,
             cancellationId,
+            null,
             MultiStageQueryContext.removeNullBytes(querySpec.getQuery().context())
         );
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
index 27779f53251..e48f1ef098a 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
@@ -44,6 +44,7 @@ import org.apache.druid.frame.processor.FileOutputChannelFactory;
 import org.apache.druid.frame.processor.FrameChannelHashPartitioner;
 import org.apache.druid.frame.processor.FrameChannelMixer;
 import org.apache.druid.frame.processor.FrameProcessor;
+import org.apache.druid.frame.processor.FrameProcessorDecorator;
 import org.apache.druid.frame.processor.FrameProcessorExecutor;
 import org.apache.druid.frame.processor.OutputChannel;
 import org.apache.druid.frame.processor.OutputChannelFactory;
@@ -62,6 +63,7 @@ import org.apache.druid.java.util.common.UOE;
 import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.msq.counters.CounterNames;
 import org.apache.druid.msq.counters.CounterTracker;
+import org.apache.druid.msq.counters.CpuCounters;
 import org.apache.druid.msq.indexing.CountingOutputChannelFactory;
 import org.apache.druid.msq.indexing.InputChannelFactory;
 import org.apache.druid.msq.indexing.InputChannelsImpl;
@@ -243,6 +245,7 @@ public class RunWorkOrder
             () -> ArenaMemoryAllocator.createOnHeap(frameContext.memoryParameters().getStandardFrameSize()),
             exec,
             cancellationId,
+            counterTracker,
             removeNullBytes
         );
 
@@ -348,7 +351,7 @@ public class RunWorkOrder
     }
 
     final ListenableFuture<ManagerReturnType> workResultFuture = exec.runAllFully(
-        processorManager,
+        counterTracker.trackCpu(processorManager, CpuCounters.LABEL_MAIN),
         maxOutstandingProcessors,
         frameContext.processorBouncer(),
         cancellationId
@@ -641,7 +644,7 @@ public class RunWorkOrder
                 );
 
             return new ResultAndChannels<>(
-                exec.runFully(mixer, cancellationId),
+                exec.runFully(counterTracker.trackCpu(mixer, CpuCounters.LABEL_MIX), cancellationId),
                 OutputChannels.wrap(Collections.singletonList(outputChannel.readOnly()))
             );
           }
@@ -723,6 +726,14 @@ public class RunWorkOrder
                 stageDefinition.getSortKey(),
                 partitionBoundariesFuture,
                 exec,
+                new FrameProcessorDecorator()
+                {
+                  @Override
+                  public <T> FrameProcessor<T> decorate(FrameProcessor<T> processor)
+                  {
+                    return counterTracker.trackCpu(processor, CpuCounters.LABEL_SORT);
+                  }
+                },
                 outputChannelFactory,
                 makeSuperSorterIntermediateOutputChannelFactory(sorterTmpDir),
                 memoryParameters.getSuperSorterMaxActiveProcessors(),
@@ -770,7 +781,11 @@ public class RunWorkOrder
                 )
             );
 
-            final ListenableFuture<Long> partitionerFuture = exec.runFully(partitioner, cancellationId);
+            final ListenableFuture<Long> partitionerFuture =
+                exec.runFully(
+                    counterTracker.trackCpu(partitioner, CpuCounters.LABEL_HASH_PARTITION),
+                    cancellationId
+                );
 
             final ResultAndChannels<Long> retVal =
                 new ResultAndChannels<>(partitionerFuture, OutputChannels.wrap(outputChannels));
@@ -844,6 +859,14 @@ public class RunWorkOrder
                         stageDefinition.getSortKey(),
                         Futures.immediateFuture(ClusterByPartitions.oneUniversalPartition()),
                         exec,
+                        new FrameProcessorDecorator()
+                        {
+                          @Override
+                          public <T> FrameProcessor<T> decorate(FrameProcessor<T> processor)
+                          {
+                            return counterTracker.trackCpu(processor, CpuCounters.LABEL_SORT);
+                          }
+                        },
                         partitionOverrideOutputChannelFactory,
                         makeSuperSorterIntermediateOutputChannelFactory(sorterTmpDir),
                         1,
@@ -929,13 +952,16 @@ public class RunWorkOrder
 
       final ListenableFuture<ClusterByStatisticsCollector> clusterByStatisticsCollectorFuture =
           exec.runAllFully(
-              ProcessorManagers.of(processors)
-                               .withAccumulation(
-                                   stageDefinition.createResultKeyStatisticsCollector(
-                                       frameContext.memoryParameters().getPartitionStatisticsMaxRetainedBytes()
+              counterTracker.trackCpu(
+                  ProcessorManagers.of(processors)
+                                   .withAccumulation(
+                                       stageDefinition.createResultKeyStatisticsCollector(
+                                           frameContext.memoryParameters().getPartitionStatisticsMaxRetainedBytes()
+                                       ),
+                                       ClusterByStatisticsCollector::addAll
                                    ),
-                                   ClusterByStatisticsCollector::addAll
-                               ),
+                  CpuCounters.LABEL_KEY_STATISTICS
+              ),
               // Run all processors simultaneously. They are lightweight and this keeps things moving.
               processors.size(),
               Bouncer.unlimited(),
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
index 90f018d07ae..92664feeabb 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerImpl.java
@@ -407,13 +407,14 @@ public class WorkerImpl implements Worker
     kernel.startReading();
 
     final QueryContext queryContext = task != null ? QueryContext.of(task.getContext()) : QueryContext.empty();
+    final boolean includeAllCounters = MultiStageQueryContext.getIncludeAllCounters(queryContext);
     final RunWorkOrder runWorkOrder = new RunWorkOrder(
         task.getControllerTaskId(),
         workOrder,
         inputChannelFactory,
         stageCounters.computeIfAbsent(
             IntObjectPair.of(workOrder.getWorkerNumber(), stageDefinition.getId()),
-            ignored -> new CounterTracker()
+            ignored -> new CounterTracker(includeAllCounters)
         ),
         workerExec,
         cancellationId,
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQIndexingModule.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQIndexingModule.java
index 4e28edc3ac1..341496f7842 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQIndexingModule.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQIndexingModule.java
@@ -27,6 +27,9 @@ import com.google.inject.Binder;
 import org.apache.druid.initialization.DruidModule;
 import org.apache.druid.msq.counters.ChannelCounters;
 import org.apache.druid.msq.counters.CounterSnapshotsSerializer;
+import org.apache.druid.msq.counters.CpuCounter;
+import org.apache.druid.msq.counters.CpuCounters;
+import org.apache.druid.msq.counters.NilQueryCounterSnapshot;
 import org.apache.druid.msq.counters.SegmentGenerationProgressCounter;
 import org.apache.druid.msq.counters.SuperSorterProgressTrackerCounter;
 import org.apache.druid.msq.counters.WarningCounters;
@@ -175,6 +178,9 @@ public class MSQIndexingModule implements DruidModule
         SuperSorterProgressTrackerCounter.Snapshot.class,
         WarningCounters.Snapshot.class,
         SegmentGenerationProgressCounter.Snapshot.class,
+        CpuCounters.Snapshot.class,
+        CpuCounter.Snapshot.class,
+        NilQueryCounterSnapshot.class,
 
         // InputSpec classes
         ExternalInputSpec.class,
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerControllerContext.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerControllerContext.java
index 0fef9d32e6d..1037aa6c2af 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerControllerContext.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerControllerContext.java
@@ -264,13 +264,15 @@ public class IndexerControllerContext implements ControllerContext
     final ImmutableMap.Builder<String, Object> taskContextOverridesBuilder = ImmutableMap.builder();
     final long maxParseExceptions = MultiStageQueryContext.getMaxParseExceptions(querySpec.getQuery().context());
     final boolean removeNullBytes = MultiStageQueryContext.removeNullBytes(querySpec.getQuery().context());
+    final boolean includeAllCounters = MultiStageQueryContext.getIncludeAllCounters(querySpec.getQuery().context());
 
     taskContextOverridesBuilder
         .put(MultiStageQueryContext.CTX_DURABLE_SHUFFLE_STORAGE, queryKernelConfig.isDurableStorage())
         .put(MSQWarnings.CTX_MAX_PARSE_EXCEPTIONS_ALLOWED, maxParseExceptions)
         .put(MultiStageQueryContext.CTX_IS_REINDEX, MSQControllerTask.isReplaceInputDataSourceTask(querySpec))
         .put(MultiStageQueryContext.CTX_MAX_CONCURRENT_STAGES, queryKernelConfig.getMaxConcurrentStages())
-        .put(MultiStageQueryContext.CTX_REMOVE_NULL_BYTES, removeNullBytes);
+        .put(MultiStageQueryContext.CTX_REMOVE_NULL_BYTES, removeNullBytes)
+        .put(MultiStageQueryContext.CTX_INCLUDE_ALL_COUNTERS, includeAllCounters);
 
     // Put the lookup loading info in the task context to facilitate selective loading of lookups.
     if (controllerTaskContext.get(LookupLoadingSpec.CTX_LOOKUP_LOADING_MODE) != null) {
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerWorkerContext.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerWorkerContext.java
index 63358467489..0b3063ef48b 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerWorkerContext.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerWorkerContext.java
@@ -80,6 +80,7 @@ public class IndexerWorkerContext implements WorkerContext
   private final ServiceClientFactory clientFactory;
   private final MemoryIntrospector memoryIntrospector;
   private final int maxConcurrentStages;
+  private final boolean includeAllCounters;
 
   @GuardedBy("this")
   private ServiceLocator controllerLocator;
@@ -105,7 +106,10 @@ public class IndexerWorkerContext implements WorkerContext
     this.clientFactory = clientFactory;
     this.memoryIntrospector = memoryIntrospector;
     this.dataServerQueryHandlerFactory = dataServerQueryHandlerFactory;
-    this.maxConcurrentStages = MultiStageQueryContext.getMaxConcurrentStages(QueryContext.of(task.getContext()));
+
+    final QueryContext queryContext = QueryContext.of(task.getContext());
+    this.maxConcurrentStages = MultiStageQueryContext.getMaxConcurrentStages(queryContext);
+    this.includeAllCounters = MultiStageQueryContext.getIncludeAllCounters(queryContext);
   }
 
   public static IndexerWorkerContext createProductionInstance(
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/InputChannelsImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/InputChannelsImpl.java
index 2c6539f5930..c1429625e99 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/InputChannelsImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/InputChannelsImpl.java
@@ -30,6 +30,8 @@ import org.apache.druid.frame.processor.FrameChannelMixer;
 import org.apache.druid.frame.processor.FrameProcessorExecutor;
 import org.apache.druid.frame.read.FrameReader;
 import org.apache.druid.frame.write.FrameWriters;
+import org.apache.druid.msq.counters.CounterTracker;
+import org.apache.druid.msq.counters.CpuCounters;
 import org.apache.druid.msq.input.stage.InputChannels;
 import org.apache.druid.msq.input.stage.ReadablePartition;
 import org.apache.druid.msq.input.stage.ReadablePartitions;
@@ -38,6 +40,7 @@ import org.apache.druid.msq.kernel.StageDefinition;
 import org.apache.druid.msq.kernel.StageId;
 import org.apache.druid.msq.kernel.StagePartition;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -55,17 +58,23 @@ public class InputChannelsImpl implements InputChannels
   private final InputChannelFactory channelFactory;
   private final Supplier<MemoryAllocator> allocatorMaker;
   private final FrameProcessorExecutor exec;
-  private final String cancellationId;
   private final Map<StagePartition, ReadablePartition> readablePartitionMap;
   private final boolean removeNullBytes;
 
+  @Nullable
+  private final String cancellationId;
+
+  @Nullable
+  private final CounterTracker counterTracker;
+
   public InputChannelsImpl(
       final QueryDefinition queryDefinition,
       final ReadablePartitions readablePartitions,
       final InputChannelFactory channelFactory,
       final Supplier<MemoryAllocator> allocatorMaker,
       final FrameProcessorExecutor exec,
-      final String cancellationId,
+      @Nullable final String cancellationId,
+      @Nullable final CounterTracker counterTracker,
       final boolean removeNullBytes
   )
   {
@@ -75,6 +84,7 @@ public class InputChannelsImpl implements InputChannels
     this.allocatorMaker = allocatorMaker;
     this.exec = exec;
     this.cancellationId = cancellationId;
+    this.counterTracker = counterTracker;
     this.removeNullBytes = removeNullBytes;
 
     for (final ReadablePartition readablePartition : readablePartitions) {
@@ -133,8 +143,6 @@ public class InputChannelsImpl implements InputChannels
           FrameWriters.makeRowBasedFrameWriterFactory(
               new SingleMemoryAllocatorFactory(allocatorMaker.get()),
               stageDefinition.getFrameReader().signature(),
-
-              // No sortColumns, because FrameChannelMerger generates frames that are sorted all on its own
               Collections.emptyList(),
               removeNullBytes
           ),
@@ -146,7 +154,10 @@ public class InputChannelsImpl implements InputChannels
       // Discard future, since there is no need to keep it. We aren't interested in its return value. If it fails,
       // downstream processors are notified through fail(e) on in-memory channels. If we need to cancel it, we use
       // the cancellationId.
-      exec.runFully(merger, cancellationId);
+      exec.runFully(
+          counterTracker == null ? merger : counterTracker.trackCpu(merger, CpuCounters.LABEL_MERGE_INPUT),
+          cancellationId
+      );
 
       return queueChannel.readable();
     }
@@ -171,7 +182,10 @@ public class InputChannelsImpl implements InputChannels
       // Discard future, since there is no need to keep it. We aren't interested in its return value. If it fails,
       // downstream processors are notified through fail(e) on in-memory channels. If we need to cancel it, we use
       // the cancellationId.
-      exec.runFully(muxer, cancellationId);
+      exec.runFully(
+          counterTracker == null ? muxer : counterTracker.trackCpu(muxer, CpuCounters.LABEL_MERGE_INPUT),
+          cancellationId
+      );
 
       return queueChannel.readable();
     }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java
index 9b715f8c8cf..ed6a7c0e7b9 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java
@@ -30,12 +30,14 @@ import org.apache.druid.error.DruidException;
 import org.apache.druid.indexing.common.TaskLockType;
 import org.apache.druid.indexing.common.task.Tasks;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.msq.counters.NilQueryCounterSnapshot;
 import org.apache.druid.msq.exec.ClusterStatisticsMergeMode;
 import org.apache.druid.msq.exec.Limits;
 import org.apache.druid.msq.exec.SegmentSource;
 import org.apache.druid.msq.indexing.destination.MSQSelectDestination;
 import org.apache.druid.msq.indexing.error.MSQWarnings;
 import org.apache.druid.msq.kernel.WorkerAssignmentStrategy;
+import org.apache.druid.msq.rpc.ControllerResource;
 import org.apache.druid.msq.sql.MSQMode;
 import org.apache.druid.query.QueryContext;
 import org.apache.druid.query.QueryContexts;
@@ -168,6 +170,19 @@ public class MultiStageQueryContext
   public static final String CTX_ARRAY_INGEST_MODE = "arrayIngestMode";
   public static final ArrayIngestMode DEFAULT_ARRAY_INGEST_MODE = ArrayIngestMode.ARRAY;
 
+  /**
+   * Whether new counters (anything other than channel, sortProgress, warnings, segmentGenerationProgress) should
+   * be included in reports. This parameter is necessary because prior to Druid 31, we lacked
+   * {@link NilQueryCounterSnapshot} as a default counter, which means that {@link SqlStatementResourceHelper} and
+   * {@link ControllerResource#httpPostCounters} would throw errors when encountering new counter types that they do
+   * not yet recognize. This causes problems during rolling updates.
+   *
+   * Once all servers are on Druid 31 or later, this can safely be flipped to "true". At that point, unknown counters
+   * are represented on the deserialization side using {@link NilQueryCounterSnapshot}.
+   */
+  public static final String CTX_INCLUDE_ALL_COUNTERS = "includeAllCounters";
+  public static final boolean DEFAULT_INCLUDE_ALL_COUNTERS = false;
+
   public static final String CTX_FORCE_TIME_SORT = DimensionsSpec.PARAMETER_FORCE_TIME_SORT;
   private static final boolean DEFAULT_FORCE_TIME_SORT = DimensionsSpec.DEFAULT_FORCE_TIME_SORT;
 
@@ -365,6 +380,14 @@ public class MultiStageQueryContext
     return queryContext.getEnum(CTX_ARRAY_INGEST_MODE, ArrayIngestMode.class, DEFAULT_ARRAY_INGEST_MODE);
   }
 
+  /**
+   * See {@link #CTX_INCLUDE_ALL_COUNTERS}.
+   */
+  public static boolean getIncludeAllCounters(final QueryContext queryContext)
+  {
+    return queryContext.getBoolean(CTX_INCLUDE_ALL_COUNTERS, DEFAULT_INCLUDE_ALL_COUNTERS);
+  }
+
   public static boolean isForceSegmentSortByTime(final QueryContext queryContext)
   {
     return queryContext.getBoolean(CTX_FORCE_TIME_SORT, DEFAULT_FORCE_TIME_SORT);
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java
index a6b0e830a20..f595ee643dd 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java
@@ -19,13 +19,20 @@
 
 package org.apache.druid.msq.counters;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableMap;
 import org.apache.druid.msq.guice.MSQIndexingModule;
 import org.apache.druid.segment.TestHelper;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.MatcherAssert;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.util.Objects;
+
 public class CountersSnapshotTreeTest
 {
   @Test
@@ -46,4 +53,93 @@ public class CountersSnapshotTreeTest
 
     Assert.assertEquals(snapshotsTree.copyMap(), snapshotsTree2.copyMap());
   }
+
+  @Test
+  public void testSerdeUnknownCounter() throws Exception
+  {
+    final ObjectMapper serializationMapper =
+        TestHelper.makeJsonMapper().registerModules(new MSQIndexingModule().getJacksonModules());
+    serializationMapper.registerSubtypes(TestCounterSnapshot.class);
+
+    final ObjectMapper deserializationMapper =
+        TestHelper.makeJsonMapper().registerModules(new MSQIndexingModule().getJacksonModules());
+
+    final TestCounter testCounter = new TestCounter(10);
+    final CounterSnapshotsTree snapshotsTree = new CounterSnapshotsTree();
+    snapshotsTree.put(1, 2, new CounterSnapshots(ImmutableMap.of("ctr", testCounter.snapshot())));
+
+    final String json = serializationMapper.writeValueAsString(snapshotsTree);
+    final CounterSnapshotsTree snapshotsTree2 = serializationMapper.readValue(json, CounterSnapshotsTree.class);
+    final CounterSnapshotsTree snapshotsTree3 = deserializationMapper.readValue(json, CounterSnapshotsTree.class);
+
+    Assert.assertEquals(snapshotsTree.copyMap(), snapshotsTree2.copyMap());
+    Assert.assertNotEquals(snapshotsTree.copyMap(), snapshotsTree3.copyMap());
+
+    // Confirm that deserializationMapper reads the TestCounterSnapshot as a NilQueryCounterSnapshot.
+    MatcherAssert.assertThat(
+        snapshotsTree3.copyMap().get(1).get(2).getMap().get("ctr"),
+        CoreMatchers.instanceOf(NilQueryCounterSnapshot.class)
+    );
+  }
+
+  private static class TestCounter implements QueryCounter
+  {
+    private final int n;
+
+    public TestCounter(int n)
+    {
+      this.n = n;
+    }
+
+    @Override
+    public QueryCounterSnapshot snapshot()
+    {
+      return new TestCounterSnapshot(n);
+    }
+  }
+
+  @JsonTypeName("test")
+  private static class TestCounterSnapshot implements QueryCounterSnapshot
+  {
+    private final int n;
+
+    @JsonCreator
+    public TestCounterSnapshot(@JsonProperty("n") int n)
+    {
+      this.n = n;
+    }
+
+    @JsonProperty("n")
+    public int getN()
+    {
+      return n;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      TestCounterSnapshot that = (TestCounterSnapshot) o;
+      return n == that.n;
+    }
+
+    @Override
+    public int hashCode()
+    {
+      return Objects.hashCode(n);
+    }
+
+    @Override
+    public String toString()
+    {
+      return "TestCounterSnapshot{" +
+             "n=" + n +
+             '}';
+    }
+  }
 }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CpuCountersTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CpuCountersTest.java
new file mode 100644
index 00000000000..d9fee44ec2d
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CpuCountersTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.counters;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class CpuCountersTest
+{
+  @Test
+  public void test_forName_runDoer()
+  {
+    final CpuCounters counters = new CpuCounters();
+    final CpuCounter counter = counters.forName("xyz");
+    counter.run(() -> { /* Nothing in particular */ });
+    final CpuCounters.Snapshot snapshot = counters.snapshot();
+    Assert.assertEquals(ImmutableSet.of("xyz"), snapshot.getCountersMap().keySet());
+  }
+
+  @Test
+  public void test_forName_runReturner()
+  {
+    final CpuCounters counters = new CpuCounters();
+    final CpuCounter counter = counters.forName("xyz");
+    Assert.assertEquals("boo", counter.run(() -> "boo"));
+    final CpuCounters.Snapshot snapshot = counters.snapshot();
+    Assert.assertEquals(ImmutableSet.of("xyz"), snapshot.getCountersMap().keySet());
+  }
+
+  @Test
+  public void test_forName_accumulate()
+  {
+    final CpuCounters counters = new CpuCounters();
+    final CpuCounter counter = counters.forName("xyz");
+    counter.accumulate(1L, 1L);
+    final CpuCounters.Snapshot snapshot = counters.snapshot();
+    Assert.assertEquals(
+        ImmutableMap.of("xyz", new CpuCounter.Snapshot(1L, 1L)),
+        snapshot.getCountersMap()
+    );
+  }
+
+  @Test
+  public void test_counter_snapshot_equals()
+  {
+    EqualsVerifier.forClass(CpuCounter.Snapshot.class)
+                  .usingGetClass()
+                  .verify();
+  }
+
+  @Test
+  public void test_counters_snapshot_equals()
+  {
+    EqualsVerifier.forClass(CpuCounters.Snapshot.class)
+                  .usingGetClass()
+                  .verify();
+  }
+
+  @Test
+  public void test_nil_equals()
+  {
+    EqualsVerifier.forClass(NilQueryCounterSnapshot.class)
+                  .usingGetClass()
+                  .verify();
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessorDecorator.java b/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessorDecorator.java
new file mode 100644
index 00000000000..f72765dc19f
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessorDecorator.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.processor;
+
+/**
+ * Passed to {@link SuperSorter} to decorate the processors it launches.
+ */
+public interface FrameProcessorDecorator
+{
+  FrameProcessorDecorator NONE = new FrameProcessorDecorator()
+  {
+    @Override
+    public <T> FrameProcessor<T> decorate(FrameProcessor<T> processor)
+    {
+      return processor;
+    }
+  };
+
+  <T> FrameProcessor<T> decorate(FrameProcessor<T> processor);
+}
diff --git a/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java b/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
index 5cb9c0830e4..d2b6934d292 100644
--- a/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
+++ b/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
@@ -131,6 +131,7 @@ public class SuperSorter
   private final List<KeyColumn> sortKey;
   private final ListenableFuture<ClusterByPartitions> outputPartitionsFuture;
   private final FrameProcessorExecutor exec;
+  private final FrameProcessorDecorator processorDecorator;
   private final OutputChannelFactory outputChannelFactory;
   private final OutputChannelFactory intermediateOutputChannelFactory;
   private final int maxChannelsPerMerger;
@@ -224,6 +225,7 @@ public class SuperSorter
       final List<KeyColumn> sortKey,
       final ListenableFuture<ClusterByPartitions> outputPartitionsFuture,
       final FrameProcessorExecutor exec,
+      final FrameProcessorDecorator processorDecorator,
       final OutputChannelFactory outputChannelFactory,
       final OutputChannelFactory intermediateOutputChannelFactory,
       final int maxActiveProcessors,
@@ -239,6 +241,7 @@ public class SuperSorter
     this.sortKey = sortKey;
     this.outputPartitionsFuture = outputPartitionsFuture;
     this.exec = exec;
+    this.processorDecorator = processorDecorator;
     this.outputChannelFactory = outputChannelFactory;
     this.intermediateOutputChannelFactory = intermediateOutputChannelFactory;
     this.maxChannelsPerMerger = maxChannelsPerMerger;
@@ -743,7 +746,7 @@ public class SuperSorter
   private <T> void runWorker(final FrameProcessor<T> worker, final Consumer<T> outConsumer)
   {
     Futures.addCallback(
-        exec.runFully(worker, cancellationId),
+        exec.runFully(processorDecorator.decorate(worker), cancellationId),
         new FutureCallback<T>()
         {
           @Override
diff --git a/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java b/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
index 2690d79c7a5..36644f6d771 100644
--- a/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
@@ -126,6 +126,7 @@ public class SuperSorterTest
           Collections.emptyList(),
           outputPartitionsFuture,
           exec,
+          FrameProcessorDecorator.NONE,
           new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
           new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
           2,
@@ -161,6 +162,7 @@ public class SuperSorterTest
           Collections.emptyList(),
           Futures.immediateFuture(ClusterByPartitions.oneUniversalPartition()),
           exec,
+          FrameProcessorDecorator.NONE,
           new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
           new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
           2,
@@ -345,6 +347,7 @@ public class SuperSorterTest
           clusterBy.getColumns(),
           clusterByPartitionsFuture,
           exec,
+          FrameProcessorDecorator.NONE,
           new FileOutputChannelFactory(tempFolder, maxBytesPerFrame, null),
           outputChannelFactory,
           maxActiveProcessors,

From 0217c8c5416fb696392a1df91d41372b5186e73d Mon Sep 17 00:00:00 2001
From: Virushade <70288012+GWphua@users.noreply.github.com>
Date: Sat, 31 Aug 2024 12:07:34 +0800
Subject: [PATCH 70/99] Change Inspection Profile to set "Method is identical
 to its super method" as error (#16976)

* Make IntelliJ's MethodIsIdenticalToSuperMethod an error

* Change codebase to follow new IntelliJ inspection

* Restore non-short-circuit boolean expressions to pass tests
---
 .idea/inspectionProfiles/Druid.xml            |  1 +
 .../parallel/LegacySinglePhaseSubTask.java    | 21 ---------
 .../tools/CustomNodeRoleClientModule.java     |  9 ----
 .../druid/testing/tools/SleepModule.java      |  9 ----
 .../apache/druid/math/expr/ConstantExpr.java  | 12 -----
 .../org/apache/druid/math/expr/Function.java  | 45 -------------------
 .../aggregation/CountBufferAggregator.java    |  7 ---
 .../aggregation/NoopBufferAggregator.java     |  7 ---
 .../epinephelinae/BufferHashGrouper.java      |  7 ---
 .../epinephelinae/RowBasedGrouperHelper.java  |  6 ---
 .../druid/segment/AutoTypeColumnMerger.java   |  5 ---
 .../segment/DictionaryMergingIterator.java    |  6 ---
 .../SimpleDictionaryMergingIterator.java      |  6 ---
 .../StringUtf8DictionaryEncodedColumn.java    |  7 ---
 .../join/table/FrameBasedIndexedTable.java    |  6 ---
 .../virtual/NestedFieldVirtualColumn.java     | 12 -----
 .../catalog/model/table/DatasourceDefn.java   |  7 ---
 17 files changed, 1 insertion(+), 172 deletions(-)

diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml
index d1f72dd2763..0889de4cc67 100644
--- a/.idea/inspectionProfiles/Druid.xml
+++ b/.idea/inspectionProfiles/Druid.xml
@@ -120,6 +120,7 @@
     <inspection_tool class="MavenDuplicatePluginInspection" enabled="true" level="ERROR" enabled_by_default="true" />
     <inspection_tool class="MavenModelInspection" enabled="true" level="WARNING" enabled_by_default="true" />
     <inspection_tool class="MethodComplexity" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="MethodIsIdenticalToSuperMethod" enabled="true" level="ERROR" enabled_by_default="true" />
     <inspection_tool class="MismatchedArrayReadWrite" enabled="true" level="ERROR" enabled_by_default="true" />
     <inspection_tool class="MismatchedCollectionQueryUpdate" enabled="true" level="ERROR" enabled_by_default="true">
       <option name="queryNames">
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/LegacySinglePhaseSubTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/LegacySinglePhaseSubTask.java
index 27a242885c3..d2694c7afd9 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/LegacySinglePhaseSubTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/LegacySinglePhaseSubTask.java
@@ -20,20 +20,11 @@
 package org.apache.druid.indexing.common.task.batch.parallel;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.ImmutableSet;
 import org.apache.druid.indexing.common.task.TaskResource;
-import org.apache.druid.server.security.Action;
-import org.apache.druid.server.security.Resource;
-import org.apache.druid.server.security.ResourceAction;
-import org.apache.druid.server.security.ResourceType;
 
-import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 public class LegacySinglePhaseSubTask extends SinglePhaseSubTask
 {
@@ -66,16 +57,4 @@ public class LegacySinglePhaseSubTask extends SinglePhaseSubTask
     return SinglePhaseSubTask.OLD_TYPE_NAME;
   }
 
-  @Nonnull
-  @JsonIgnore
-  @Override
-  public Set<ResourceAction> getInputSourceResources()
-  {
-    return getIngestionSchema().getIOConfig().getInputSource() != null ?
-           getIngestionSchema().getIOConfig().getInputSource().getTypes()
-                               .stream()
-                               .map(i -> new ResourceAction(new Resource(i, ResourceType.EXTERNAL), Action.READ))
-                               .collect(Collectors.toSet()) :
-           ImmutableSet.of();
-  }
 }
diff --git a/integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/CustomNodeRoleClientModule.java b/integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/CustomNodeRoleClientModule.java
index f1171ffa4a6..df73a8c7d80 100644
--- a/integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/CustomNodeRoleClientModule.java
+++ b/integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/CustomNodeRoleClientModule.java
@@ -19,13 +19,9 @@
 
 package org.apache.druid.testing.tools;
 
-import com.fasterxml.jackson.databind.Module;
 import com.google.inject.Binder;
 import org.apache.druid.initialization.DruidModule;
 
-import java.util.Collections;
-import java.util.List;
-
 /**
  * Super-simple "client" for the custom node role which defines
  * the node role so that REST APIs and the system tables are
@@ -40,9 +36,4 @@ public class CustomNodeRoleClientModule implements DruidModule
     // NodeRoles.addRole(binder, CliCustomNodeRole.NODE_ROLE);
   }
 
-  @Override
-  public List<? extends Module> getJacksonModules()
-  {
-    return Collections.emptyList();
-  }
 }
diff --git a/integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/SleepModule.java b/integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/SleepModule.java
index a8028f6920f..6e7362b3b38 100644
--- a/integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/SleepModule.java
+++ b/integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/SleepModule.java
@@ -19,22 +19,13 @@
 
 package org.apache.druid.testing.tools;
 
-import com.fasterxml.jackson.databind.Module;
 import com.google.inject.Binder;
 import org.apache.druid.guice.ExpressionModule;
 import org.apache.druid.initialization.DruidModule;
 import org.apache.druid.sql.guice.SqlBindings;
 
-import java.util.Collections;
-import java.util.List;
-
 public class SleepModule implements DruidModule
 {
-  @Override
-  public List<? extends Module> getJacksonModules()
-  {
-    return Collections.emptyList();
-  }
 
   @Override
   public void configure(Binder binder)
diff --git a/processing/src/main/java/org/apache/druid/math/expr/ConstantExpr.java b/processing/src/main/java/org/apache/druid/math/expr/ConstantExpr.java
index 85cebd478ee..f6007512f6c 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/ConstantExpr.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/ConstantExpr.java
@@ -246,12 +246,6 @@ class LongExpr extends ConstantExpr<Long>
     super(ExpressionType.LONG, Preconditions.checkNotNull(value, "value"));
   }
 
-  @Override
-  public String toString()
-  {
-    return String.valueOf(value);
-  }
-
   @Override
   protected ExprEval realEval()
   {
@@ -329,12 +323,6 @@ class DoubleExpr extends ConstantExpr<Double>
     super(ExpressionType.DOUBLE, Preconditions.checkNotNull(value, "value"));
   }
 
-  @Override
-  public String toString()
-  {
-    return String.valueOf(value);
-  }
-
   @Override
   protected ExprEval realEval()
   {
diff --git a/processing/src/main/java/org/apache/druid/math/expr/Function.java b/processing/src/main/java/org/apache/druid/math/expr/Function.java
index a7fc7f8c186..3f0a6e7aa9b 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/Function.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/Function.java
@@ -530,11 +530,6 @@ public interface Function extends NamedFunction
    */
   abstract class ArraysMergeFunction extends ArraysFunction
   {
-    @Override
-    public Set<Expr> getArrayInputs(List<Expr> args)
-    {
-      return ImmutableSet.copyOf(args);
-    }
 
     @Override
     public boolean hasArrayOutput()
@@ -1183,16 +1178,6 @@ public interface Function extends NamedFunction
       return NAME;
     }
 
-    @Nullable
-    @Override
-    public ExpressionType getOutputType(Expr.InputBindingInspector inspector, List<Expr> args)
-    {
-      return ExpressionTypeConversion.function(
-          args.get(0).getOutputType(inspector),
-          args.get(1).getOutputType(inspector)
-      );
-    }
-
     @Override
     public boolean canVectorize(Expr.InputBindingInspector inspector, List<Expr> args)
     {
@@ -2315,18 +2300,6 @@ public interface Function extends NamedFunction
       return ExprEval.ofLongBoolean(!super.apply(args, bindings).asBoolean());
     }
 
-    @Override
-    public void validateArguments(List<Expr> args)
-    {
-      validationHelperCheckArgumentCount(args, 2);
-    }
-
-    @Nullable
-    @Override
-    public ExpressionType getOutputType(Expr.InputBindingInspector inspector, List<Expr> args)
-    {
-      return ExpressionType.LONG;
-    }
   }
 
   /**
@@ -3399,18 +3372,6 @@ public interface Function extends NamedFunction
       return ExprEval.ofArray(arrayType, out);
     }
 
-    @Override
-    public Set<Expr> getScalarInputs(List<Expr> args)
-    {
-      return ImmutableSet.copyOf(args);
-    }
-
-    @Override
-    public Set<Expr> getArrayInputs(List<Expr> args)
-    {
-      return Collections.emptySet();
-    }
-
     @Override
     public boolean hasArrayOutput()
     {
@@ -3544,12 +3505,6 @@ public interface Function extends NamedFunction
       return ExprEval.ofStringArray(arrayString.split(split != null ? split : ""));
     }
 
-    @Override
-    public Set<Expr> getScalarInputs(List<Expr> args)
-    {
-      return ImmutableSet.copyOf(args);
-    }
-
     @Override
     public boolean hasArrayOutput()
     {
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/CountBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/CountBufferAggregator.java
index 7f30fa0006e..7b966d6f1e1 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/CountBufferAggregator.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/CountBufferAggregator.java
@@ -19,8 +19,6 @@
 
 package org.apache.druid.query.aggregation;
 
-import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
-
 import java.nio.ByteBuffer;
 
 /**
@@ -71,9 +69,4 @@ public class CountBufferAggregator implements BufferAggregator
     // no resources to cleanup
   }
 
-  @Override
-  public void inspectRuntimeShape(RuntimeShapeInspector inspector)
-  {
-    // nothing to inspect
-  }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/NoopBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/NoopBufferAggregator.java
index 95774203d03..ab348273e3b 100644
--- a/processing/src/main/java/org/apache/druid/query/aggregation/NoopBufferAggregator.java
+++ b/processing/src/main/java/org/apache/druid/query/aggregation/NoopBufferAggregator.java
@@ -19,8 +19,6 @@
 
 package org.apache.druid.query.aggregation;
 
-import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
-
 import java.nio.ByteBuffer;
 
 public final class NoopBufferAggregator implements BufferAggregator
@@ -75,9 +73,4 @@ public final class NoopBufferAggregator implements BufferAggregator
   {
   }
 
-  @Override
-  public void inspectRuntimeShape(RuntimeShapeInspector inspector)
-  {
-    // nothing to inspect
-  }
 }
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouper.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouper.java
index c4d04697716..182a4857760 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouper.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouper.java
@@ -32,7 +32,6 @@ import java.util.AbstractList;
 import java.util.Collections;
 import java.util.List;
 import java.util.NoSuchElementException;
-import java.util.function.ToIntFunction;
 
 public class BufferHashGrouper<KeyType> extends AbstractBufferHashGrouper<KeyType>
 {
@@ -128,12 +127,6 @@ public class BufferHashGrouper<KeyType> extends AbstractBufferHashGrouper<KeyTyp
     return initialized;
   }
 
-  @Override
-  public ToIntFunction<KeyType> hashFunction()
-  {
-    return Groupers::hashObject;
-  }
-
   @Override
   public void newBucketHook(int bucketOffset)
   {
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
index 0e73d5db6f4..8c42bb935d8 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
@@ -1815,12 +1815,6 @@ public class RowBasedGrouperHelper
                 );
       }
 
-      @Override
-      public int getKeyBufferValueSize()
-      {
-        return Integer.BYTES;
-      }
-
       @Override
       public BufferComparator getBufferComparator()
       {
diff --git a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java
index 801eaf112a5..7ce9709f30f 100644
--- a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java
+++ b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java
@@ -445,11 +445,6 @@ public class AutoTypeColumnMerger implements DimensionMergerV9
       return counter;
     }
 
-    @Override
-    public void remove()
-    {
-      throw new UnsupportedOperationException("remove");
-    }
   }
 
   public static class IdLookupArrayIterator implements Iterator<int[]>
diff --git a/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java b/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java
index 2e570c7d3de..bfbfe1aee38 100644
--- a/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java
+++ b/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java
@@ -162,12 +162,6 @@ public class DictionaryMergingIterator<T extends Comparable<T>> implements Close
     return value;
   }
 
-  @Override
-  public void remove()
-  {
-    throw new UnsupportedOperationException("remove");
-  }
-
   @Override
   public void close()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/SimpleDictionaryMergingIterator.java b/processing/src/main/java/org/apache/druid/segment/SimpleDictionaryMergingIterator.java
index d38506fe724..75c47f70766 100644
--- a/processing/src/main/java/org/apache/druid/segment/SimpleDictionaryMergingIterator.java
+++ b/processing/src/main/java/org/apache/druid/segment/SimpleDictionaryMergingIterator.java
@@ -106,12 +106,6 @@ public class SimpleDictionaryMergingIterator<T extends Comparable<T>> implements
     return counter;
   }
 
-  @Override
-  public void remove()
-  {
-    throw new UnsupportedOperationException("remove");
-  }
-
   @Override
   public void close()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringUtf8DictionaryEncodedColumn.java b/processing/src/main/java/org/apache/druid/segment/column/StringUtf8DictionaryEncodedColumn.java
index c3ebde1854c..a5966096a6e 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/StringUtf8DictionaryEncodedColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/StringUtf8DictionaryEncodedColumn.java
@@ -222,13 +222,6 @@ public class StringUtf8DictionaryEncodedColumn implements DictionaryEncodedColum
           return DimensionSelectorUtils.makeValueMatcherGeneric(this, predicateFactory);
         }
 
-        @Nullable
-        @Override
-        public Object getObject()
-        {
-          return defaultGetObject();
-        }
-
         @Override
         public Class classOfObject()
         {
diff --git a/processing/src/main/java/org/apache/druid/segment/join/table/FrameBasedIndexedTable.java b/processing/src/main/java/org/apache/druid/segment/join/table/FrameBasedIndexedTable.java
index 080a3c920ae..ceaffe846e2 100644
--- a/processing/src/main/java/org/apache/druid/segment/join/table/FrameBasedIndexedTable.java
+++ b/processing/src/main/java/org/apache/druid/segment/join/table/FrameBasedIndexedTable.java
@@ -263,12 +263,6 @@ public class FrameBasedIndexedTable implements IndexedTable
     };
   }
 
-  @Override
-  public boolean isCacheable()
-  {
-    return false;
-  }
-
   @Override
   public void close()
   {
diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
index 160415924a3..aa7dd8b88b7 100644
--- a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
@@ -1423,13 +1423,6 @@ public class NestedFieldVirtualColumn implements VirtualColumn
       return Numbers.tryParseLong(o, 0L);
     }
 
-    @Override
-    public void inspectRuntimeShape(RuntimeShapeInspector inspector)
-    {
-      inspector.visit("baseSelector", baseSelector);
-      inspector.visit("parts", parts);
-    }
-
     @Override
     public boolean isNull()
     {
@@ -1461,11 +1454,6 @@ public class NestedFieldVirtualColumn implements VirtualColumn
       return null;
     }
 
-    @Override
-    public Class<?> classOfObject()
-    {
-      return Object.class;
-    }
   }
 
   /**
diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java
index b3dc953cf8e..cca497f0b7f 100644
--- a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java
+++ b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java
@@ -21,7 +21,6 @@ package org.apache.druid.catalog.model.table;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.druid.catalog.model.CatalogUtils;
 import org.apache.druid.catalog.model.ColumnSpec;
 import org.apache.druid.catalog.model.Columns;
 import org.apache.druid.catalog.model.ModelProperties;
@@ -78,12 +77,6 @@ public class DatasourceDefn extends TableDefn
       super(SEGMENT_GRANULARITY_PROPERTY);
     }
 
-    @Override
-    public void validate(Object value, ObjectMapper jsonMapper)
-    {
-      String gran = decode(value, jsonMapper);
-      CatalogUtils.validateGranularity(gran);
-    }
   }
 
   public static class HiddenColumnsDefn extends StringListPropertyDefn

From 6eb42e8d5a550b562445ce677d6fb83ee52f7103 Mon Sep 17 00:00:00 2001
From: Parag Jain <pjain1@apache.org>
Date: Sun, 1 Sep 2024 00:25:55 +0530
Subject: [PATCH 71/99] fix extraction of timeseries results from result level
 cache (#16895)

* fix extraction of timeseries results from result level cache

* remove unneded import

* add test
---
 .../TimeseriesQueryQueryToolChest.java        |  7 ++++
 .../TimeseriesQueryQueryToolChestTest.java    |  4 +-
 .../sql/calcite/CalciteSelectQueryTest.java   | 41 +++++++++++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java
index 05b25d05c21..18bcc713dec 100644
--- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java
+++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java
@@ -383,6 +383,13 @@ public class TimeseriesQueryQueryToolChest extends QueryToolChest<Result<Timeser
               timestamp = granularity.toDateTime(Preconditions.checkNotNull(timestampNumber, "timestamp").longValue());
             }
 
+            // If "timestampResultField" is set, we must include a copy of the timestamp in the result.
+            // This is used by the SQL layer when it generates a Timeseries query for a group-by-time-floor SQL query.
+            // The SQL layer expects the result of the time-floor to have a specific name that is not going to be "__time".
+            if (StringUtils.isNotEmpty(query.getTimestampResultField()) && timestamp != null) {
+              retVal.put(query.getTimestampResultField(), timestamp.getMillis());
+            }
+
             CacheStrategy.fetchAggregatorsFromCache(
                 aggs,
                 resultIter,
diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
index f1afabe9c75..8d9166d2746 100644
--- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
+++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
@@ -97,7 +97,7 @@ public class TimeseriesQueryQueryToolChestTest
                 ),
                 ImmutableList.of(new ConstantPostAggregator("post", 10)),
                 0,
-                null
+                ImmutableMap.of(TimeseriesQuery.CTX_TIMESTAMP_RESULT_FIELD, "ts_field")
             )
         );
 
@@ -106,6 +106,7 @@ public class TimeseriesQueryQueryToolChestTest
         DateTimes.utc(123L),
         new TimeseriesResultValue(
             ImmutableMap.of(
+                "ts_field", 123L,
                 "metric1", 2,
                 "metric0", 3,
                 "complexMetric", new SerializablePairLongString(123L, "val1")
@@ -130,6 +131,7 @@ public class TimeseriesQueryQueryToolChestTest
         DateTimes.utc(123L),
         new TimeseriesResultValue(
             ImmutableMap.of(
+                "ts_field", 123L,
                 "metric1", 2,
                 "metric0", 3,
                 "complexMetric", "val1",
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
index 6e06c5d4b5d..973b1eb4df1 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
@@ -2118,6 +2118,47 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
         .run();
   }
 
+  @SqlTestFrameworkConfig.ResultCache(ResultCacheMode.ENABLED)
+  @Test
+  public void testTimseriesResultCachePullConsistency()
+  {
+    skipVectorize();
+
+    String query = "SELECT \n"
+                   + "  (t1.\"__time\") AS \"__time\", \n"
+                   + "  (ANY_VALUE(t1.\"added\")) AS \"added\" \n"
+                   + "FROM \n"
+                   + "  (\n"
+                   + "    SELECT \n"
+                   + "      (time_floor(\"__time\", 'PT1H')) AS \"__time\", \n"
+                   + "      (SUM(added)) AS \"added\" \n"
+                   + "    FROM \"wikipedia\" \n"
+                   + "    GROUP BY 1 \n"
+                   + "    ORDER BY \"__time\" \n"
+                   + "    LIMIT 1\n"
+                   + "  ) t1 \n"
+                   + "GROUP BY 1 \n"
+                   + "ORDER BY \"__time\"";
+
+    testBuilder()
+        .sql(query)
+        .expectedResults(
+            ImmutableList.of(
+                new Object[]{1442016000000L, 32251L}
+            )
+        )
+        .run();
+
+    testBuilder()
+        .sql(query)
+        .expectedResults(
+            ImmutableList.of(
+                new Object[]{1442016000000L, 32251L}
+            )
+        )
+        .run();
+  }
+
   @Test
   public void testSqlToRelInConversion()
   {

From fe3d589ff9f9ca82b6c12f619eb62fe52b2542ab Mon Sep 17 00:00:00 2001
From: Kashif Faraz <kashif.faraz@gmail.com>
Date: Sun, 1 Sep 2024 19:23:13 -0700
Subject: [PATCH 72/99] Run compaction as a supervisor on Overlord (#16768)

Description
-----------
Auto-compaction currently poses several challenges as it:
1. may get stuck on a failing interval.
2. may get stuck on the latest interval if more data keeps coming into it.
3. always picks the latest interval regardless of the level of compaction in it.
4. may never pick a datasource if its intervals are not very recent.
5. requires setting an explicit period which does not cater to the changing needs of a Druid cluster.

This PR introduces various improvements to compaction scheduling to tackle the above problems.

Change Summary
--------------
1. Run compaction for a datasource as a supervisor of type `autocompact` on Overlord.
2. Make compaction policy extensible and configurable.
3. Track status of recently submitted compaction tasks and pass this info to policy.
4. Add `/simulate` API on both Coordinator and Overlord to run compaction simulations.
5. Redirect compaction status APIs to the Overlord when compaction supervisors are enabled.
---
 .../NewestSegmentFirstPolicyBenchmark.java    |   18 +-
 .../client/ControllerChatHandlerTest.java     |    6 +-
 .../apache/druid/guice/SupervisorModule.java  |   54 +
 .../indexing/compact/CompactionScheduler.java |   65 +
 .../compact/CompactionSupervisor.java         |  156 ++
 .../compact/CompactionSupervisorSpec.java     |  115 ++
 .../indexing/compact/LocalOverlordClient.java |  263 +++
 .../compact/OverlordCompactionScheduler.java  |  357 ++++
 .../indexing/overlord/DruidOverlord.java      |    4 +
 .../indexing/overlord/TaskQueryTool.java      |   81 +-
 .../http/OverlordCompactionResource.java      |  150 ++
 .../overlord/http/OverlordResource.java       |   13 +-
 ...ionBasedPublishedSegmentRetrieverTest.java |   15 +-
 .../actions/SegmentAllocationQueueTest.java   |   54 +-
 .../batch/parallel/PartialCompactionTest.java |    3 -
 .../ConcurrentReplaceAndAppendTest.java       |   11 +-
 ...ncurrentReplaceAndStreamingAppendTest.java |   17 +-
 .../compact/CompactionSupervisorSpecTest.java |  226 +++
 .../OverlordCompactionSchedulerTest.java      |  372 ++++
 .../overlord/http/OverlordResourceTest.java   |    2 +-
 .../indexing/overlord/http/OverlordTest.java  |    2 +
 .../druid/java/util/common/Intervals.java     |    5 -
 .../java/util/common/guava/Comparators.java   |   18 +
 .../util/common/guava/ComparatorsTest.java    |   19 +
 .../apache/druid/segment/TestDataSource.java  |   19 +-
 .../druid/timeline/SegmentTimelineTest.java   |    3 +-
 .../metadata/SegmentsMetadataManager.java     |    4 -
 .../metadata/SqlSegmentsMetadataManager.java  |    4 +-
 .../druid/rpc/indexing/OverlordClient.java    |   26 +
 .../rpc/indexing/OverlordClientImpl.java      |   47 +
 .../compaction/BaseCandidateSearchPolicy.java |  101 ++
 .../CompactionCandidate.java}                 |   88 +-
 .../CompactionCandidateSearchPolicy.java      |   54 +
 .../compaction/CompactionRunSimulator.java    |  311 ++++
 .../CompactionSegmentIterator.java}           |   30 +-
 .../compaction/CompactionSimulateResult.java  |   44 +
 .../CompactionStatistics.java                 |    2 +-
 .../CompactionStatus.java                     |  168 +-
 .../compaction/CompactionStatusTracker.java   |  242 +++
 .../compaction/CompactionTaskStatus.java      |   56 +
 .../DataSourceCompactibleSegmentIterator.java |  229 +--
 .../compaction/NewestSegmentFirstPolicy.java  |   49 +
 ...riorityBasedCompactionSegmentIterator.java |   54 +-
 .../apache/druid/server/compaction/Table.java |   85 +
 .../coordinator/AutoCompactionSnapshot.java   |   43 +-
 .../coordinator/ClusterCompactionConfig.java  |   22 +-
 .../CompactionSupervisorConfig.java           |   85 +
 .../DataSourceCompactionConfig.java           |    9 +
 .../coordinator/DruidCompactionConfig.java    |   31 +-
 .../server/coordinator/DruidCoordinator.java  |   76 +-
 .../compact/CompactionSegmentIterator.java    |   49 -
 .../compact/NewestSegmentFirstPolicy.java     |   62 -
 .../coordinator/duty/CompactSegments.java     |  156 +-
 .../duty/CoordinatorDutyUtils.java            |   11 +-
 .../coordinator/duty/KillUnusedSegments.java  |    2 +-
 .../CoordinatorCompactionConfigsResource.java |    1 +
 ...ava => CoordinatorCompactionResource.java} |   85 +-
 .../client/indexing/NoopOverlordClient.java   |   19 +
 ...exerSQLMetadataStorageCoordinatorTest.java |   57 +-
 ...SqlMetadataStorageCoordinatorTestBase.java |   19 +-
 .../metadata/SQLMetadataRuleManagerTest.java  |   43 +-
 .../SqlSegmentsMetadataManagerTest.java       |  225 ++-
 .../rpc/indexing/OverlordClientImplTest.java  |   80 +
 .../CompactionRunSimulatorTest.java           |  165 ++
 .../compaction/CompactionStatusTest.java      |  355 ++++
 .../CompactionStatusTrackerTest.java          |  124 ++
 ...aSourceCompactibleSegmentIteratorTest.java |   10 +-
 .../NewestSegmentFirstPolicyTest.java         | 1588 +++++++----------
 .../AutoCompactionSnapshotTest.java           |    2 +-
 .../coordinator/CreateDataSegments.java       |   34 +-
 ...aSourceCompactionConfigAuditEntryTest.java |   24 +-
 ...DataSourceCompactionConfigHistoryTest.java |   17 +-
 .../DruidCompactionConfigTest.java            |   35 +-
 .../coordinator/DruidCoordinatorTest.java     |   35 +-
 .../balancer/CostBalancerStrategyTest.java    |   27 +-
 .../balancer/ReservoirSegmentSamplerTest.java |    3 +-
 .../balancer/SegmentToMoveCalculatorTest.java |    5 +-
 .../compact/CompactionStatusTest.java         |  178 --
 .../coordinator/duty/CompactSegmentsTest.java |  141 +-
 .../duty/KillStalePendingSegmentsTest.java    |   73 +-
 .../loading/RoundRobinServerSelectorTest.java |    3 +-
 .../rules/BroadcastDistributionRuleTest.java  |   26 +-
 .../coordinator/rules/LoadRuleTest.java       |   53 +-
 .../simulate/BalancingStrategiesTest.java     |    7 +-
 .../CoordinatorSimulationBaseTest.java        |   19 +-
 .../CoordinatorSimulationBuilder.java         |    9 +-
 .../simulate/MarkSegmentsAsUnusedTest.java    |    3 +-
 .../simulate/RoundRobinAssignmentTest.java    |   11 +-
 .../simulate/SegmentBalancingTest.java        |    5 +-
 .../simulate/SegmentLoadingTest.java          |    7 +-
 .../simulate/TestSegmentsMetadataManager.java |   13 +-
 ...rdinatorCompactionConfigsResourceTest.java |   69 +-
 ...=> CoordinatorCompactionResourceTest.java} |   81 +-
 .../org/apache/druid/cli/CliCoordinator.java  |   11 +-
 .../org/apache/druid/cli/CliOverlord.java     |   28 +-
 .../security/SecurityResourceFilterTest.java  |    4 +-
 96 files changed, 5740 insertions(+), 2142 deletions(-)
 create mode 100644 indexing-service/src/main/java/org/apache/druid/guice/SupervisorModule.java
 create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionScheduler.java
 create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionSupervisor.java
 create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionSupervisorSpec.java
 create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/compact/LocalOverlordClient.java
 create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
 create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResource.java
 create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/compact/CompactionSupervisorSpecTest.java
 create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java
 rename server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorModule.java => processing/src/test/java/org/apache/druid/segment/TestDataSource.java (68%)
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java
 rename server/src/main/java/org/apache/druid/server/{coordinator/compact/SegmentsToCompact.java => compaction/CompactionCandidate.java} (55%)
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
 rename server/src/main/java/org/apache/druid/server/{coordinator/compact/CompactionSegmentSearchPolicy.java => compaction/CompactionSegmentIterator.java} (56%)
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionSimulateResult.java
 rename server/src/main/java/org/apache/druid/server/{coordinator/compact => compaction}/CompactionStatistics.java (96%)
 rename server/src/main/java/org/apache/druid/server/{coordinator/compact => compaction}/CompactionStatus.java (70%)
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionTaskStatus.java
 rename server/src/main/java/org/apache/druid/server/{coordinator/compact => compaction}/DataSourceCompactibleSegmentIterator.java (73%)
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicy.java
 rename server/src/main/java/org/apache/druid/server/{coordinator/compact => compaction}/PriorityBasedCompactionSegmentIterator.java (70%)
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/Table.java
 create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/CompactionSupervisorConfig.java
 delete mode 100644 server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionSegmentIterator.java
 delete mode 100644 server/src/main/java/org/apache/druid/server/coordinator/compact/NewestSegmentFirstPolicy.java
 rename server/src/main/java/org/apache/druid/server/http/{CompactionResource.java => CoordinatorCompactionResource.java} (53%)
 create mode 100644 server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
 create mode 100644 server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java
 create mode 100644 server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java
 rename server/src/test/java/org/apache/druid/server/{coordinator/compact => compaction}/DataSourceCompactibleSegmentIteratorTest.java (88%)
 rename server/src/test/java/org/apache/druid/server/{coordinator/compact => compaction}/NewestSegmentFirstPolicyTest.java (52%)
 delete mode 100644 server/src/test/java/org/apache/druid/server/coordinator/compact/CompactionStatusTest.java
 rename server/src/test/java/org/apache/druid/server/http/{CompactionResourceTest.java => CoordinatorCompactionResourceTest.java} (53%)

diff --git a/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java b/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java
index 87e92ab6fb1..fb938dca4cc 100644
--- a/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java
@@ -24,9 +24,11 @@ import com.google.common.collect.ImmutableMap;
 import org.apache.druid.client.DataSourcesSnapshot;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.server.coordinator.compact.CompactionSegmentIterator;
-import org.apache.druid.server.coordinator.compact.CompactionSegmentSearchPolicy;
-import org.apache.druid.server.coordinator.compact.NewestSegmentFirstPolicy;
+import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy;
+import org.apache.druid.server.compaction.CompactionSegmentIterator;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
+import org.apache.druid.server.compaction.NewestSegmentFirstPolicy;
+import org.apache.druid.server.compaction.PriorityBasedCompactionSegmentIterator;
 import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.SegmentTimeline;
 import org.apache.druid.timeline.partition.NumberedShardSpec;
@@ -61,7 +63,7 @@ public class NewestSegmentFirstPolicyBenchmark
 {
   private static final String DATA_SOURCE_PREFIX = "dataSource_";
 
-  private final CompactionSegmentSearchPolicy policy = new NewestSegmentFirstPolicy(new DefaultObjectMapper());
+  private final CompactionCandidateSearchPolicy policy = new NewestSegmentFirstPolicy(null);
 
   @Param("100")
   private int numDataSources;
@@ -132,7 +134,13 @@ public class NewestSegmentFirstPolicyBenchmark
   @Benchmark
   public void measureNewestSegmentFirstPolicy(Blackhole blackhole)
   {
-    final CompactionSegmentIterator iterator = policy.createIterator(compactionConfigs, dataSources, Collections.emptyMap());
+    final CompactionSegmentIterator iterator = new PriorityBasedCompactionSegmentIterator(
+        policy,
+        compactionConfigs,
+        dataSources,
+        Collections.emptyMap(),
+        new CompactionStatusTracker(new DefaultObjectMapper())
+    );
     for (int i = 0; i < numCompactionTaskSlots && iterator.hasNext(); i++) {
       blackhole.consume(iterator.next());
     }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/client/ControllerChatHandlerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/client/ControllerChatHandlerTest.java
index dd04814d879..1701d43c1cf 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/client/ControllerChatHandlerTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/client/ControllerChatHandlerTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.msq.indexing.client;
 import org.apache.druid.indexer.report.KillTaskReport;
 import org.apache.druid.indexer.report.TaskReport;
 import org.apache.druid.msq.exec.Controller;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.security.AuthorizerMapper;
 import org.junit.Assert;
 import org.junit.Test;
@@ -33,8 +34,6 @@ import javax.ws.rs.core.Response;
 
 public class ControllerChatHandlerTest
 {
-  private static final String DATASOURCE = "wiki";
-
   @Test
   public void testHttpGetLiveReports()
   {
@@ -47,7 +46,8 @@ public class ControllerChatHandlerTest
            .thenReturn(reportMap);
 
     final AuthorizerMapper authorizerMapper = new AuthorizerMapper(null);
-    ControllerChatHandler chatHandler = new ControllerChatHandler(controller, DATASOURCE, authorizerMapper);
+    ControllerChatHandler chatHandler
+        = new ControllerChatHandler(controller, TestDataSource.WIKI, authorizerMapper);
 
     HttpServletRequest httpRequest = Mockito.mock(HttpServletRequest.class);
     Mockito.when(httpRequest.getAttribute(ArgumentMatchers.anyString()))
diff --git a/indexing-service/src/main/java/org/apache/druid/guice/SupervisorModule.java b/indexing-service/src/main/java/org/apache/druid/guice/SupervisorModule.java
new file mode 100644
index 00000000000..f855459159a
--- /dev/null
+++ b/indexing-service/src/main/java/org/apache/druid/guice/SupervisorModule.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.druid.guice;
+
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Binder;
+import org.apache.druid.indexing.compact.CompactionSupervisorSpec;
+import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig;
+import org.apache.druid.initialization.DruidModule;
+import org.apache.druid.server.coordinator.CompactionSupervisorConfig;
+
+import java.util.List;
+
+public class SupervisorModule implements DruidModule
+{
+  @Override
+  public void configure(Binder binder)
+  {
+    JsonConfigProvider.bind(binder, "druid.supervisor", SupervisorStateManagerConfig.class);
+    JsonConfigProvider.bind(binder, "druid.supervisor.compaction", CompactionSupervisorConfig.class);
+  }
+
+  @Override
+  public List<? extends Module> getJacksonModules()
+  {
+    return ImmutableList.of(
+        new SimpleModule(getClass().getSimpleName())
+            .registerSubtypes(
+                new NamedType(CompactionSupervisorSpec.class, CompactionSupervisorSpec.TYPE)
+            )
+    );
+  }
+}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionScheduler.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionScheduler.java
new file mode 100644
index 00000000000..83a87b358c9
--- /dev/null
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionScheduler.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.compact;
+
+import org.apache.druid.server.compaction.CompactionSimulateResult;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.coordinator.ClusterCompactionConfig;
+import org.apache.druid.server.coordinator.CompactionConfigValidationResult;
+import org.apache.druid.server.coordinator.CompactionSupervisorConfig;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+
+import java.util.Map;
+
+/**
+ * Compaction scheduler that runs on the Overlord if {@link CompactionSupervisorConfig}
+ * is enabled.
+ * <p>
+ * Usage:
+ * <ul>
+ * <li>When an active {@link CompactionSupervisor} starts, it should register
+ * itself by calling {@link #startCompaction}.</li>
+ * <li>When a suspended {@link CompactionSupervisor} starts, it should stop
+ * compaction by calling {@link #stopCompaction}.</li>
+ * <li>When stopping, any {@link CompactionSupervisor} (active or suspended)
+ * should call {@link #stopCompaction}.</li>
+ * </ul>
+ */
+public interface CompactionScheduler
+{
+  void start();
+
+  void stop();
+
+  boolean isRunning();
+
+  CompactionConfigValidationResult validateCompactionConfig(DataSourceCompactionConfig compactionConfig);
+
+  void startCompaction(String dataSourceName, DataSourceCompactionConfig compactionConfig);
+
+  void stopCompaction(String dataSourceName);
+
+  Map<String, AutoCompactionSnapshot> getAllCompactionSnapshots();
+
+  AutoCompactionSnapshot getCompactionSnapshot(String dataSource);
+
+  CompactionSimulateResult simulateRunWithConfigUpdate(ClusterCompactionConfig updateRequest);
+
+}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionSupervisor.java
new file mode 100644
index 00000000000..d4a3c46ba0b
--- /dev/null
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionSupervisor.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.compact;
+
+import org.apache.druid.indexing.overlord.DataSourceMetadata;
+import org.apache.druid.indexing.overlord.supervisor.Supervisor;
+import org.apache.druid.indexing.overlord.supervisor.SupervisorReport;
+import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager;
+import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+
+/**
+ * Supervisor for compaction of a single datasource.
+ */
+public class CompactionSupervisor implements Supervisor
+{
+  private static final Logger log = new Logger(CompactionSupervisor.class);
+
+  private final String dataSource;
+  private final CompactionScheduler scheduler;
+  private final CompactionSupervisorSpec supervisorSpec;
+
+  public CompactionSupervisor(
+      CompactionSupervisorSpec supervisorSpec,
+      CompactionScheduler scheduler
+  )
+  {
+    this.supervisorSpec = supervisorSpec;
+    this.scheduler = scheduler;
+    this.dataSource = supervisorSpec.getSpec().getDataSource();
+  }
+
+  @Override
+  public void start()
+  {
+    if (supervisorSpec.isSuspended()) {
+      log.info("Suspending compaction for dataSource[%s].", dataSource);
+      scheduler.stopCompaction(dataSource);
+    } else {
+      log.info("Starting compaction for dataSource[%s].", dataSource);
+      scheduler.startCompaction(dataSource, supervisorSpec.getSpec());
+    }
+  }
+
+  @Override
+  public void stop(boolean stopGracefully)
+  {
+    log.info("Stopping compaction for dataSource[%s].", dataSource);
+    scheduler.stopCompaction(dataSource);
+  }
+
+  @Override
+  public SupervisorReport<AutoCompactionSnapshot> getStatus()
+  {
+    final AutoCompactionSnapshot snapshot;
+    if (supervisorSpec.isSuspended()) {
+      snapshot = AutoCompactionSnapshot.builder(dataSource)
+                                       .withStatus(AutoCompactionSnapshot.AutoCompactionScheduleStatus.NOT_ENABLED)
+                                       .build();
+    } else {
+      snapshot = scheduler.getCompactionSnapshot(dataSource);
+    }
+
+    return new SupervisorReport<>(supervisorSpec.getId(), DateTimes.nowUtc(), snapshot);
+  }
+
+  @Override
+  public SupervisorStateManager.State getState()
+  {
+    if (!scheduler.isRunning()) {
+      return State.SCHEDULER_STOPPED;
+    } else if (supervisorSpec.isSuspended()) {
+      return State.SUSPENDED;
+    } else {
+      return State.RUNNING;
+    }
+  }
+
+  // Un-implemented methods used only by streaming supervisors
+
+  @Override
+  public void reset(DataSourceMetadata dataSourceMetadata)
+  {
+    throw new UnsupportedOperationException("Resetting not supported for 'autocompact' supervisors.");
+  }
+
+  @Override
+  public void resetOffsets(DataSourceMetadata resetDataSourceMetadata)
+  {
+    throw new UnsupportedOperationException("Resetting offsets not supported for 'autocompact' supervisors.");
+  }
+
+  @Override
+  public void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata)
+  {
+    throw new UnsupportedOperationException("Checkpointing not supported for 'autocompact' supervisors.");
+  }
+
+  @Override
+  public LagStats computeLagStats()
+  {
+    throw new UnsupportedOperationException("Lag stats not supported for 'autocompact' supervisors.");
+  }
+
+  @Override
+  public int getActiveTaskGroupsCount()
+  {
+    throw new UnsupportedOperationException("Task groups not supported for 'autocompact' supervisors.");
+  }
+
+  public enum State implements SupervisorStateManager.State
+  {
+    SCHEDULER_STOPPED(true),
+    RUNNING(true),
+    SUSPENDED(true),
+    UNHEALTHY(false);
+
+    private final boolean healthy;
+
+    State(boolean healthy)
+    {
+      this.healthy = healthy;
+    }
+
+    @Override
+    public boolean isFirstRunOnly()
+    {
+      return false;
+    }
+
+    @Override
+    public boolean isHealthy()
+    {
+      return healthy;
+    }
+  }
+}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionSupervisorSpec.java
new file mode 100644
index 00000000000..6911f35f96e
--- /dev/null
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionSupervisorSpec.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.compact;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.druid.common.config.Configs;
+import org.apache.druid.error.InvalidInput;
+import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec;
+import org.apache.druid.server.coordinator.CompactionConfigValidationResult;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+
+import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
+
+public class CompactionSupervisorSpec implements SupervisorSpec
+{
+  public static final String TYPE = "autocompact";
+  public static final String ID_PREFIX = "autocompact__";
+
+  private final boolean suspended;
+  private final DataSourceCompactionConfig spec;
+  private final CompactionScheduler scheduler;
+
+  @JsonCreator
+  public CompactionSupervisorSpec(
+      @JsonProperty("spec") DataSourceCompactionConfig spec,
+      @JsonProperty("suspended") @Nullable Boolean suspended,
+      @JacksonInject CompactionScheduler scheduler
+  )
+  {
+    final CompactionConfigValidationResult validationResult = scheduler.validateCompactionConfig(spec);
+    if (!validationResult.isValid()) {
+      throw InvalidInput.exception("Compaction supervisor 'spec' is invalid. Reason[%s].", validationResult.getReason());
+    }
+
+    this.spec = spec;
+    this.suspended = Configs.valueOrDefault(suspended, false);
+    this.scheduler = scheduler;
+  }
+
+  @JsonProperty
+  public DataSourceCompactionConfig getSpec()
+  {
+    return spec;
+  }
+
+  @Override
+  @JsonProperty
+  public boolean isSuspended()
+  {
+    return suspended;
+  }
+
+  @Override
+  public String getId()
+  {
+    return ID_PREFIX + spec.getDataSource();
+  }
+
+  @Override
+  public CompactionSupervisor createSupervisor()
+  {
+    return new CompactionSupervisor(this, scheduler);
+  }
+
+  @Override
+  public List<String> getDataSources()
+  {
+    return Collections.singletonList(spec.getDataSource());
+  }
+
+  @Override
+  public CompactionSupervisorSpec createSuspendedSpec()
+  {
+    return new CompactionSupervisorSpec(spec, true, scheduler);
+  }
+
+  @Override
+  public CompactionSupervisorSpec createRunningSpec()
+  {
+    return new CompactionSupervisorSpec(spec, false, scheduler);
+  }
+
+  @Override
+  public String getType()
+  {
+    return TYPE;
+  }
+
+  @Override
+  public String getSource()
+  {
+    return "";
+  }
+}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/LocalOverlordClient.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/LocalOverlordClient.java
new file mode 100644
index 00000000000..eeaf138c106
--- /dev/null
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/LocalOverlordClient.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.compact;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.druid.client.indexing.ClientCompactionTaskQuery;
+import org.apache.druid.client.indexing.IndexingTotalWorkerCapacityInfo;
+import org.apache.druid.client.indexing.IndexingWorkerInfo;
+import org.apache.druid.client.indexing.TaskPayloadResponse;
+import org.apache.druid.client.indexing.TaskStatusResponse;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.indexer.TaskStatus;
+import org.apache.druid.indexer.TaskStatusPlus;
+import org.apache.druid.indexer.report.TaskReport;
+import org.apache.druid.indexing.common.task.CompactionTask;
+import org.apache.druid.indexing.overlord.TaskMaster;
+import org.apache.druid.indexing.overlord.TaskQueryTool;
+import org.apache.druid.indexing.overlord.TaskQueue;
+import org.apache.druid.indexing.overlord.http.TotalWorkerCapacityResponse;
+import org.apache.druid.indexing.overlord.supervisor.SupervisorStatus;
+import org.apache.druid.java.util.common.CloseableIterators;
+import org.apache.druid.java.util.common.concurrent.Execs;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.java.util.common.parsers.CloseableIterator;
+import org.apache.druid.metadata.LockFilterPolicy;
+import org.apache.druid.rpc.ServiceRetryPolicy;
+import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.joda.time.Interval;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
+/**
+ * Dummy Overlord client used by the {@link OverlordCompactionScheduler} to fetch
+ * task related info. This client simply redirects all queries to the
+ * {@link TaskQueryTool} and all updates to the {@link TaskQueue}.
+ */
+class LocalOverlordClient implements OverlordClient
+{
+  private static final Logger log = new Logger(LocalOverlordClient.class);
+
+  private final TaskMaster taskMaster;
+  private final TaskQueryTool taskQueryTool;
+  private final ObjectMapper objectMapper;
+
+  LocalOverlordClient(TaskMaster taskMaster, TaskQueryTool taskQueryTool, ObjectMapper objectMapper)
+  {
+    this.taskMaster = taskMaster;
+    this.taskQueryTool = taskQueryTool;
+    this.objectMapper = objectMapper;
+  }
+
+  @Override
+  public ListenableFuture<Void> runTask(String taskId, Object clientTaskQuery)
+  {
+    return futureOf(() -> {
+      getValidTaskQueue().add(
+          convertTask(clientTaskQuery, ClientCompactionTaskQuery.class, CompactionTask.class)
+      );
+      return null;
+    });
+  }
+
+  @Override
+  public ListenableFuture<Void> cancelTask(String taskId)
+  {
+    return futureOf(() -> {
+      getValidTaskQueue().shutdown(taskId, "Shutdown by Compaction Scheduler");
+      return null;
+    });
+  }
+
+  @Override
+  public ListenableFuture<TaskPayloadResponse> taskPayload(String taskId)
+  {
+    ClientCompactionTaskQuery taskPayload = taskQueryTool.getTask(taskId).transform(
+        task -> convertTask(task, CompactionTask.class, ClientCompactionTaskQuery.class)
+    ).orNull();
+    return futureOf(() -> new TaskPayloadResponse(taskId, taskPayload));
+  }
+
+  @Override
+  public ListenableFuture<CloseableIterator<TaskStatusPlus>> taskStatuses(
+      @Nullable String state,
+      @Nullable String dataSource,
+      @Nullable Integer maxCompletedTasks
+  )
+  {
+    final ListenableFuture<List<TaskStatusPlus>> tasksFuture
+        = futureOf(taskQueryTool::getAllActiveTasks);
+    return Futures.transform(
+        tasksFuture,
+        taskList -> CloseableIterators.withEmptyBaggage(taskList.iterator()),
+        Execs.directExecutor()
+    );
+  }
+
+  @Override
+  public ListenableFuture<Map<String, TaskStatus>> taskStatuses(Set<String> taskIds)
+  {
+    return futureOf(() -> taskQueryTool.getMultipleTaskStatuses(taskIds));
+  }
+
+  @Override
+  public ListenableFuture<Map<String, List<Interval>>> findLockedIntervals(
+      List<LockFilterPolicy> lockFilterPolicies
+  )
+  {
+    return futureOf(() -> taskQueryTool.getLockedIntervals(lockFilterPolicies));
+  }
+
+  @Override
+  public ListenableFuture<IndexingTotalWorkerCapacityInfo> getTotalWorkerCapacity()
+  {
+    return futureOf(() -> convert(taskQueryTool.getTotalWorkerCapacity()));
+  }
+
+  private TaskQueue getValidTaskQueue()
+  {
+    Optional<TaskQueue> taskQueue = taskMaster.getTaskQueue();
+    if (taskQueue.isPresent()) {
+      return taskQueue.get();
+    } else {
+      throw DruidException.defensive("No TaskQueue. Cannot proceed.");
+    }
+  }
+
+  private <T> ListenableFuture<T> futureOf(Supplier<T> supplier)
+  {
+    try {
+      return Futures.immediateFuture(supplier.get());
+    }
+    catch (Exception e) {
+      return Futures.immediateFailedFuture(e);
+    }
+  }
+
+  private IndexingTotalWorkerCapacityInfo convert(TotalWorkerCapacityResponse capacity)
+  {
+    if (capacity == null) {
+      return null;
+    } else {
+      return new IndexingTotalWorkerCapacityInfo(
+          capacity.getCurrentClusterCapacity(),
+          capacity.getMaximumCapacityWithAutoScale()
+      );
+    }
+  }
+
+  private <U, V> V convertTask(Object taskPayload, Class<U> inputType, Class<V> outputType)
+  {
+    if (taskPayload == null) {
+      return null;
+    } else if (!inputType.isInstance(taskPayload)) {
+      throw DruidException.defensive(
+          "Unknown type[%s] for compaction task. Expected type[%s].",
+          taskPayload.getClass().getSimpleName(), inputType.getSimpleName()
+      );
+    }
+
+    try {
+      return objectMapper.readValue(
+          objectMapper.writeValueAsBytes(taskPayload),
+          outputType
+      );
+    }
+    catch (IOException e) {
+      log.warn(e, "Could not convert task[%s] to client compatible object", taskPayload);
+      throw DruidException.defensive(
+          "Could not convert task[%s] to compatible object.",
+          taskPayload
+      );
+    }
+  }
+
+  // Unsupported methods as these are not used by the CompactionScheduler / CompactSegments duty
+
+  @Override
+  public ListenableFuture<URI> findCurrentLeader()
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<TaskStatusResponse> taskStatus(String taskId)
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<TaskReport.ReportMap> taskReportAsMap(String taskId)
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<CloseableIterator<SupervisorStatus>> supervisorStatuses()
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<Integer> killPendingSegments(String dataSource, Interval interval)
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<List<IndexingWorkerInfo>> getWorkers()
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource)
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<Boolean> isCompactionSupervisorEnabled()
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public OverlordClient withRetryPolicy(ServiceRetryPolicy retryPolicy)
+  {
+    return this;
+  }
+}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
new file mode 100644
index 00000000000..e7b6440deb6
--- /dev/null
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.compact;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+import com.google.inject.Inject;
+import org.apache.druid.client.indexing.ClientCompactionRunnerInfo;
+import org.apache.druid.indexer.TaskLocation;
+import org.apache.druid.indexer.TaskStatus;
+import org.apache.druid.indexing.overlord.TaskMaster;
+import org.apache.druid.indexing.overlord.TaskQueryTool;
+import org.apache.druid.indexing.overlord.TaskRunner;
+import org.apache.druid.indexing.overlord.TaskRunnerListener;
+import org.apache.druid.java.util.common.Stopwatch;
+import org.apache.druid.java.util.common.concurrent.Execs;
+import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
+import org.apache.druid.metadata.SegmentsMetadataManager;
+import org.apache.druid.server.compaction.CompactionRunSimulator;
+import org.apache.druid.server.compaction.CompactionSimulateResult;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.coordinator.ClusterCompactionConfig;
+import org.apache.druid.server.coordinator.CompactionConfigValidationResult;
+import org.apache.druid.server.coordinator.CompactionSupervisorConfig;
+import org.apache.druid.server.coordinator.CoordinatorOverlordServiceConfig;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+import org.apache.druid.server.coordinator.DruidCompactionConfig;
+import org.apache.druid.server.coordinator.duty.CompactSegments;
+import org.apache.druid.server.coordinator.stats.CoordinatorRunStats;
+import org.apache.druid.server.coordinator.stats.CoordinatorStat;
+import org.apache.druid.server.coordinator.stats.Dimension;
+import org.apache.druid.server.coordinator.stats.Stats;
+import org.apache.druid.timeline.SegmentTimeline;
+import org.joda.time.Duration;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Implementation of {@link CompactionScheduler}.
+ * <p>
+ * When {@link CompactionSupervisorConfig} is enabled, this class performs the
+ * following responsibilities on the leader Overlord:
+ * <ul>
+ * <li>Poll segments from metadata</li>
+ * <li>Check compaction status every 5 seconds</li>
+ * <li>Submit compaction tasks for active datasources</li>
+ * <li>Track status of submitted compaction tasks</li>
+ * </ul>
+ * Internally, this class uses an instance of {@link CompactSegments} duty.
+ */
+public class OverlordCompactionScheduler implements CompactionScheduler
+{
+  private static final Logger log = new Logger(OverlordCompactionScheduler.class);
+
+  private static final long SCHEDULE_PERIOD_SECONDS = 5;
+  private static final Duration METRIC_EMISSION_PERIOD = Duration.standardMinutes(5);
+
+  private final SegmentsMetadataManager segmentManager;
+  private final LocalOverlordClient overlordClient;
+  private final ServiceEmitter emitter;
+  private final TaskMaster taskMaster;
+
+  private final CompactionSupervisorConfig supervisorConfig;
+  private final Supplier<DruidCompactionConfig> compactionConfigSupplier;
+  private final ConcurrentHashMap<String, DataSourceCompactionConfig> activeDatasourceConfigs;
+
+  /**
+   * Single-threaded executor to process the compaction queue.
+   */
+  private final ScheduledExecutorService executor;
+
+  private final CompactionStatusTracker statusTracker;
+
+  /**
+   * Listener to watch task completion events and update CompactionStatusTracker.
+   * This helps in avoiding unnecessary metadata store calls.
+   */
+  private final TaskRunnerListener taskRunnerListener;
+
+  private final AtomicBoolean started = new AtomicBoolean(false);
+  private final CompactSegments duty;
+
+  /**
+   * The scheduler should enable/disable polling of segments only if the Overlord
+   * is running in standalone mode, otherwise this is handled by the DruidCoordinator
+   * class itself.
+   */
+  private final boolean shouldPollSegments;
+
+  private final Stopwatch sinceStatsEmitted = Stopwatch.createUnstarted();
+
+  @Inject
+  public OverlordCompactionScheduler(
+      TaskMaster taskMaster,
+      TaskQueryTool taskQueryTool,
+      SegmentsMetadataManager segmentManager,
+      Supplier<DruidCompactionConfig> compactionConfigSupplier,
+      CompactionStatusTracker statusTracker,
+      CompactionSupervisorConfig supervisorConfig,
+      CoordinatorOverlordServiceConfig coordinatorOverlordServiceConfig,
+      ScheduledExecutorFactory executorFactory,
+      ServiceEmitter emitter,
+      ObjectMapper objectMapper
+  )
+  {
+    this.segmentManager = segmentManager;
+    this.emitter = emitter;
+    this.taskMaster = taskMaster;
+    this.supervisorConfig = supervisorConfig;
+    this.compactionConfigSupplier = compactionConfigSupplier;
+
+    this.executor = executorFactory.create(1, "CompactionScheduler-%s");
+    this.statusTracker = statusTracker;
+    this.shouldPollSegments = segmentManager != null
+                              && !coordinatorOverlordServiceConfig.isEnabled();
+    this.overlordClient = new LocalOverlordClient(taskMaster, taskQueryTool, objectMapper);
+    this.duty = new CompactSegments(this.statusTracker, overlordClient);
+    this.activeDatasourceConfigs = new ConcurrentHashMap<>();
+
+    this.taskRunnerListener = new TaskRunnerListener()
+    {
+      @Override
+      public String getListenerId()
+      {
+        return "OverlordCompactionScheduler";
+      }
+
+      @Override
+      public void locationChanged(String taskId, TaskLocation newLocation)
+      {
+        // Do nothing
+      }
+
+      @Override
+      public void statusChanged(String taskId, TaskStatus status)
+      {
+        if (status.isComplete()) {
+          statusTracker.onTaskFinished(taskId, status);
+        }
+      }
+    };
+  }
+
+  @Override
+  public void start()
+  {
+    if (isEnabled() && started.compareAndSet(false, true)) {
+      log.info("Starting compaction scheduler.");
+      initState();
+      scheduleOnExecutor(this::scheduledRun);
+    }
+  }
+
+  @Override
+  public void stop()
+  {
+    if (isEnabled() && started.compareAndSet(true, false)) {
+      log.info("Stopping compaction scheduler.");
+      cleanupState();
+    }
+  }
+
+  @Override
+  public boolean isRunning()
+  {
+    return isEnabled() && started.get();
+  }
+
+  @Override
+  public CompactionConfigValidationResult validateCompactionConfig(DataSourceCompactionConfig compactionConfig)
+  {
+    if (compactionConfig == null) {
+      return CompactionConfigValidationResult.failure("Cannot be null");
+    } else {
+      return ClientCompactionRunnerInfo.validateCompactionConfig(
+          compactionConfig,
+          compactionConfigSupplier.get().getEngine()
+      );
+    }
+  }
+
+  @Override
+  public void startCompaction(String dataSourceName, DataSourceCompactionConfig config)
+  {
+    // Track active datasources even if scheduler has not started yet because
+    // SupervisorManager is started before the scheduler
+    if (isEnabled()) {
+      activeDatasourceConfigs.put(dataSourceName, config);
+    }
+  }
+
+  @Override
+  public void stopCompaction(String dataSourceName)
+  {
+    activeDatasourceConfigs.remove(dataSourceName);
+    statusTracker.removeDatasource(dataSourceName);
+  }
+
+  private synchronized void initState()
+  {
+    final Optional<TaskRunner> taskRunnerOptional = taskMaster.getTaskRunner();
+    if (taskRunnerOptional.isPresent()) {
+      taskRunnerOptional.get().registerListener(taskRunnerListener, Execs.directExecutor());
+    }
+    if (shouldPollSegments) {
+      segmentManager.startPollingDatabasePeriodically();
+    }
+  }
+
+  private synchronized void cleanupState()
+  {
+    final Optional<TaskRunner> taskRunnerOptional = taskMaster.getTaskRunner();
+    if (taskRunnerOptional.isPresent()) {
+      taskRunnerOptional.get().unregisterListener(taskRunnerListener.getListenerId());
+    }
+    statusTracker.stop();
+    activeDatasourceConfigs.clear();
+
+    if (shouldPollSegments) {
+      segmentManager.stopPollingDatabasePeriodically();
+    }
+  }
+
+  private boolean isEnabled()
+  {
+    return supervisorConfig.isEnabled();
+  }
+
+  private synchronized void scheduledRun()
+  {
+    if (isRunning()) {
+      try {
+        runCompactionDuty();
+      }
+      catch (Exception e) {
+        log.error(e, "Error processing compaction queue. Continuing schedule.");
+      }
+      scheduleOnExecutor(this::scheduledRun);
+    } else {
+      cleanupState();
+    }
+  }
+
+  private synchronized void runCompactionDuty()
+  {
+    final CoordinatorRunStats stats = new CoordinatorRunStats();
+    duty.run(getLatestConfig(), getCurrentDatasourceTimelines(), stats);
+
+    // Emit stats only if emission period has elapsed
+    if (!sinceStatsEmitted.isRunning() || sinceStatsEmitted.hasElapsed(METRIC_EMISSION_PERIOD)) {
+      stats.forEachStat(
+          (stat, dimensions, value) -> {
+            if (stat.shouldEmit()) {
+              emitStat(stat, dimensions.getValues(), value);
+            }
+          }
+      );
+      sinceStatsEmitted.restart();
+    } else {
+      // Always emit number of submitted tasks
+      long numSubmittedTasks = stats.get(Stats.Compaction.SUBMITTED_TASKS);
+      emitStat(Stats.Compaction.SUBMITTED_TASKS, Collections.emptyMap(), numSubmittedTasks);
+    }
+  }
+
+  @Override
+  public AutoCompactionSnapshot getCompactionSnapshot(String dataSource)
+  {
+    return duty.getAutoCompactionSnapshot(dataSource);
+  }
+
+  @Override
+  public Map<String, AutoCompactionSnapshot> getAllCompactionSnapshots()
+  {
+    return duty.getAutoCompactionSnapshot();
+  }
+
+  @Override
+  public CompactionSimulateResult simulateRunWithConfigUpdate(ClusterCompactionConfig updateRequest)
+  {
+    if (isRunning()) {
+      return new CompactionRunSimulator(statusTracker, overlordClient).simulateRunWithConfig(
+          getLatestConfig().withClusterConfig(updateRequest),
+          getCurrentDatasourceTimelines()
+      );
+    } else {
+      return new CompactionSimulateResult(Collections.emptyMap());
+    }
+  }
+
+  private void emitStat(CoordinatorStat stat, Map<Dimension, String> dimensionValues, long value)
+  {
+    ServiceMetricEvent.Builder eventBuilder = new ServiceMetricEvent.Builder();
+    dimensionValues.forEach(
+        (dim, dimValue) -> eventBuilder.setDimension(dim.reportedName(), dimValue)
+    );
+    emitter.emit(eventBuilder.setMetric(stat.getMetricName(), value));
+  }
+
+  private DruidCompactionConfig getLatestConfig()
+  {
+    return DruidCompactionConfig
+        .empty()
+        .withClusterConfig(compactionConfigSupplier.get().clusterConfig())
+        .withDatasourceConfigs(new ArrayList<>(activeDatasourceConfigs.values()));
+  }
+
+  private Map<String, SegmentTimeline> getCurrentDatasourceTimelines()
+  {
+    return segmentManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+                         .getUsedSegmentsTimelinesPerDataSource();
+  }
+
+  private void scheduleOnExecutor(Runnable runnable)
+  {
+    executor.schedule(
+        () -> {
+          try {
+            runnable.run();
+          }
+          catch (Throwable t) {
+            log.error(t, "Error while executing runnable");
+          }
+        },
+        SCHEDULE_PERIOD_SECONDS,
+        TimeUnit.SECONDS
+    );
+  }
+}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/DruidOverlord.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/DruidOverlord.java
index f99189d3589..fb4730e7df4 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/DruidOverlord.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/DruidOverlord.java
@@ -29,6 +29,7 @@ import org.apache.druid.guice.annotations.Self;
 import org.apache.druid.indexing.common.actions.SegmentAllocationQueue;
 import org.apache.druid.indexing.common.actions.TaskActionClientFactory;
 import org.apache.druid.indexing.common.task.TaskContextEnricher;
+import org.apache.druid.indexing.compact.CompactionScheduler;
 import org.apache.druid.indexing.overlord.config.DefaultTaskConfig;
 import org.apache.druid.indexing.overlord.config.TaskLockConfig;
 import org.apache.druid.indexing.overlord.config.TaskQueueConfig;
@@ -87,6 +88,7 @@ public class DruidOverlord
       final OverlordDutyExecutor overlordDutyExecutor,
       @IndexingService final DruidLeaderSelector overlordLeaderSelector,
       final SegmentAllocationQueue segmentAllocationQueue,
+      final CompactionScheduler compactionScheduler,
       final ObjectMapper mapper,
       final TaskContextEnricher taskContextEnricher
   )
@@ -140,6 +142,7 @@ public class DruidOverlord
                 {
                   segmentAllocationQueue.becomeLeader();
                   taskMaster.becomeLeader(taskRunner, taskQueue);
+                  compactionScheduler.start();
 
                   // Announce the node only after all the services have been initialized
                   initialized = true;
@@ -150,6 +153,7 @@ public class DruidOverlord
                 public void stop()
                 {
                   serviceAnnouncer.unannounce(node);
+                  compactionScheduler.stop();
                   taskMaster.stopBeingLeader();
                   segmentAllocationQueue.stopBeingLeader();
                 }
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueryTool.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueryTool.java
index b25bde067c7..4c37af7ef16 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueryTool.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueryTool.java
@@ -20,10 +20,11 @@
 package org.apache.druid.indexing.overlord;
 
 import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import com.google.inject.Inject;
-import org.apache.druid.common.config.JacksonConfigManager;
 import org.apache.druid.indexer.TaskInfo;
 import org.apache.druid.indexer.TaskStatus;
 import org.apache.druid.indexer.TaskStatusPlus;
@@ -34,6 +35,7 @@ import org.apache.druid.indexing.overlord.http.TaskStateLookup;
 import org.apache.druid.indexing.overlord.http.TotalWorkerCapacityResponse;
 import org.apache.druid.indexing.overlord.setup.DefaultWorkerBehaviorConfig;
 import org.apache.druid.indexing.overlord.setup.WorkerBehaviorConfig;
+import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
@@ -41,6 +43,7 @@ import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.metadata.TaskLookup;
 import org.apache.druid.metadata.TaskLookup.TaskLookupType;
+import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.joda.time.Interval;
 
@@ -50,6 +53,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -66,7 +70,7 @@ public class TaskQueryTool
   private final TaskStorage storage;
   private final TaskLockbox taskLockbox;
   private final TaskMaster taskMaster;
-  private final JacksonConfigManager configManager;
+  private final Supplier<WorkerBehaviorConfig> workerBehaviorConfigSupplier;
   private final ProvisioningStrategy provisioningStrategy;
 
   @Inject
@@ -75,13 +79,13 @@ public class TaskQueryTool
       TaskLockbox taskLockbox,
       TaskMaster taskMaster,
       ProvisioningStrategy provisioningStrategy,
-      JacksonConfigManager configManager
+      Supplier<WorkerBehaviorConfig> workerBehaviorConfigSupplier
   )
   {
     this.storage = storage;
     this.taskLockbox = taskLockbox;
     this.taskMaster = taskMaster;
-    this.configManager = configManager;
+    this.workerBehaviorConfigSupplier = workerBehaviorConfigSupplier;
     this.provisioningStrategy = provisioningStrategy;
   }
 
@@ -108,12 +112,17 @@ public class TaskQueryTool
     return storage.getTaskInfos(TaskLookup.activeTasksOnly(), dataSource);
   }
 
-  private List<TaskStatusPlus> getTaskStatusPlusList(
-      Map<TaskLookupType, TaskLookup> taskLookups,
-      @Nullable String dataSource
-  )
+  public Map<String, TaskStatus> getMultipleTaskStatuses(Set<String> taskIds)
   {
-    return storage.getTaskStatusPlusList(taskLookups, dataSource);
+    final Map<String, TaskStatus> result = Maps.newHashMapWithExpectedSize(taskIds.size());
+    for (String taskId : taskIds) {
+      final Optional<TaskStatus> optional = getTaskStatus(taskId);
+      if (optional.isPresent()) {
+        result.put(taskId, optional.get());
+      }
+    }
+
+    return result;
   }
 
   public Optional<Task> getTask(final String taskId)
@@ -144,6 +153,45 @@ public class TaskQueryTool
     return storage.getTaskInfo(taskId);
   }
 
+  public List<TaskStatusPlus> getAllActiveTasks()
+  {
+    final Optional<TaskQueue> taskQueue = taskMaster.getTaskQueue();
+    if (taskQueue.isPresent()) {
+      // Serve active task statuses from memory
+      final List<TaskStatusPlus> taskStatusPlusList = new ArrayList<>();
+
+      // Use a dummy created time as this is not used by the caller, just needs to be non-null
+      final DateTime createdTime = DateTimes.nowUtc();
+
+      final List<Task> activeTasks = taskQueue.get().getTasks();
+      for (Task task : activeTasks) {
+        final Optional<TaskStatus> statusOptional = taskQueue.get().getTaskStatus(task.getId());
+        if (statusOptional.isPresent()) {
+          final TaskStatus status = statusOptional.get();
+          taskStatusPlusList.add(
+              new TaskStatusPlus(
+                  task.getId(),
+                  task.getGroupId(),
+                  task.getType(),
+                  createdTime,
+                  createdTime,
+                  status.getStatusCode(),
+                  null,
+                  status.getDuration(),
+                  status.getLocation(),
+                  task.getDataSource(),
+                  status.getErrorMsg()
+              )
+          );
+        }
+      }
+
+      return taskStatusPlusList;
+    } else {
+      return getTaskStatusPlusList(TaskStateLookup.ALL, null, null, 0, null);
+    }
+  }
+
   public List<TaskStatusPlus> getTaskStatusPlusList(
       TaskStateLookup state,
       @Nullable String dataSource,
@@ -172,7 +220,7 @@ public class TaskQueryTool
     // This way, we can use the snapshot from taskStorage as the source of truth for the set of tasks to process
     // and use the snapshot from taskRunner as a reference for potential task state updates happened
     // after the first snapshotting.
-    Stream<TaskStatusPlus> taskStatusPlusStream = getTaskStatusPlusList(
+    Stream<TaskStatusPlus> taskStatusPlusStream = getTaskStatusPlusStream(
         state,
         dataSource,
         createdTimeDuration,
@@ -238,7 +286,7 @@ public class TaskQueryTool
     return taskStatuses;
   }
 
-  private Stream<TaskStatusPlus> getTaskStatusPlusList(
+  private Stream<TaskStatusPlus> getTaskStatusPlusStream(
       TaskStateLookup state,
       @Nullable String dataSource,
       Duration createdTimeDuration,
@@ -274,10 +322,8 @@ public class TaskQueryTool
         throw new IAE("Unknown state: [%s]", state);
     }
 
-    final Stream<TaskStatusPlus> taskStatusPlusStream = getTaskStatusPlusList(
-        taskLookups,
-        dataSource
-    ).stream();
+    final Stream<TaskStatusPlus> taskStatusPlusStream
+        = storage.getTaskStatusPlusList(taskLookups, dataSource).stream();
     if (type != null) {
       return taskStatusPlusStream.filter(
           statusPlus -> type.equals(statusPlus == null ? null : statusPlus.getType())
@@ -369,10 +415,7 @@ public class TaskQueryTool
 
   public WorkerBehaviorConfig getLatestWorkerConfig()
   {
-    return configManager.watch(
-        WorkerBehaviorConfig.CONFIG_KEY,
-        WorkerBehaviorConfig.class
-    ).get();
+    return workerBehaviorConfigSupplier.get();
   }
 
 }
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResource.java
new file mode 100644
index 00000000000..9f21f2ec4c1
--- /dev/null
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResource.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.http;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import com.sun.jersey.spi.container.ResourceFilters;
+import org.apache.druid.indexing.compact.CompactionScheduler;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.coordinator.ClusterCompactionConfig;
+import org.apache.druid.server.coordinator.CompactionSupervisorConfig;
+import org.apache.druid.server.http.security.StateResourceFilter;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Contains the same logic as {@code CompactionResource} but the APIs are served
+ * by {@link CompactionScheduler} instead of {@code DruidCoordinator}.
+ */
+@Path("/druid/indexer/v1/compaction")
+public class OverlordCompactionResource
+{
+  private final CompactionScheduler scheduler;
+  private final CompactionSupervisorConfig supervisorConfig;
+
+  @Inject
+  public OverlordCompactionResource(
+      CompactionSupervisorConfig supervisorConfig,
+      CompactionScheduler scheduler
+  )
+  {
+    this.scheduler = scheduler;
+    this.supervisorConfig = supervisorConfig;
+  }
+
+  @GET
+  @Path("/isSupervisorEnabled")
+  @Produces(MediaType.APPLICATION_JSON)
+  @ResourceFilters(StateResourceFilter.class)
+  public Response isCompactionSupervisorEnabled()
+  {
+    return Response.ok(supervisorConfig.isEnabled()).build();
+  }
+
+  @GET
+  @Path("/progress")
+  @Produces(MediaType.APPLICATION_JSON)
+  @ResourceFilters(StateResourceFilter.class)
+  public Response getCompactionProgress(
+      @QueryParam("dataSource") String dataSource
+  )
+  {
+    if (!supervisorConfig.isEnabled()) {
+      return buildErrorResponseIfSchedulerDisabled();
+    }
+
+    if (dataSource == null || dataSource.isEmpty()) {
+      return Response.status(Response.Status.BAD_REQUEST)
+                     .entity(Collections.singletonMap("error", "No DataSource specified"))
+                     .build();
+    }
+
+    final AutoCompactionSnapshot snapshot = scheduler.getCompactionSnapshot(dataSource);
+    if (snapshot == null) {
+      return Response.status(Response.Status.NOT_FOUND)
+                     .entity(Collections.singletonMap("error", "Unknown DataSource"))
+                     .build();
+    } else {
+      return Response.ok(Collections.singletonMap("remainingSegmentSize", snapshot.getBytesAwaitingCompaction()))
+                     .build();
+    }
+  }
+
+  @GET
+  @Path("/status")
+  @Produces(MediaType.APPLICATION_JSON)
+  @ResourceFilters(StateResourceFilter.class)
+  public Response getCompactionSnapshots(
+      @QueryParam("dataSource") String dataSource
+  )
+  {
+    if (!supervisorConfig.isEnabled()) {
+      return buildErrorResponseIfSchedulerDisabled();
+    }
+
+    final Collection<AutoCompactionSnapshot> snapshots;
+    if (dataSource == null || dataSource.isEmpty()) {
+      snapshots = scheduler.getAllCompactionSnapshots().values();
+    } else {
+      AutoCompactionSnapshot autoCompactionSnapshot = scheduler.getCompactionSnapshot(dataSource);
+      if (autoCompactionSnapshot == null) {
+        return Response.status(Response.Status.NOT_FOUND)
+                       .entity(Collections.singletonMap("error", "Unknown DataSource"))
+                       .build();
+      }
+      snapshots = Collections.singleton(autoCompactionSnapshot);
+    }
+    return Response.ok(Collections.singletonMap("latestStatus", snapshots)).build();
+  }
+
+  @POST
+  @Path("/simulate")
+  @Consumes(MediaType.APPLICATION_JSON)
+  @ResourceFilters(StateResourceFilter.class)
+  public Response simulateRunWithConfigUpdate(
+      ClusterCompactionConfig updatePayload
+  )
+  {
+    return Response.ok().entity(
+        scheduler.simulateRunWithConfigUpdate(updatePayload)
+    ).build();
+  }
+
+  private Response buildErrorResponseIfSchedulerDisabled()
+  {
+    return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(
+        ImmutableMap.of(
+            "error",
+            "Compaction Supervisors are disabled on the Overlord."
+            + " Use Coordinator APIs to fetch compaction status."
+        )
+    ).build();
+  }
+}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java
index b62e1de055f..84bf260b6f9 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java
@@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 import com.google.inject.Inject;
 import com.sun.jersey.spi.container.ResourceFilters;
 import org.apache.druid.audit.AuditEntry;
@@ -418,17 +417,9 @@ public class OverlordResource
   {
     if (CollectionUtils.isNullOrEmpty(taskIds)) {
       return Response.status(Response.Status.BAD_REQUEST).entity("No Task IDs provided.").build();
+    } else {
+      return Response.ok().entity(taskQueryTool.getMultipleTaskStatuses(taskIds)).build();
     }
-
-    final Map<String, TaskStatus> result = Maps.newHashMapWithExpectedSize(taskIds.size());
-    for (String taskId : taskIds) {
-      final Optional<TaskStatus> optional = taskQueryTool.getTaskStatus(taskId);
-      if (optional.isPresent()) {
-        result.put(taskId, optional.get());
-      }
-    }
-
-    return Response.ok().entity(result).build();
   }
 
   @GET
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/appenderator/ActionBasedPublishedSegmentRetrieverTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/appenderator/ActionBasedPublishedSegmentRetrieverTest.java
index dd1cd5c59ca..744d357f4e8 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/appenderator/ActionBasedPublishedSegmentRetrieverTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/appenderator/ActionBasedPublishedSegmentRetrieverTest.java
@@ -28,6 +28,7 @@ import org.apache.druid.indexing.common.actions.TaskActionClient;
 import org.apache.druid.indexing.overlord.Segments;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordinator.CreateDataSegments;
 import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.SegmentId;
@@ -59,7 +60,7 @@ public class ActionBasedPublishedSegmentRetrieverTest
   public void testRetrieveSegmentsById() throws IOException
   {
     final List<DataSegment> segments =
-        CreateDataSegments.ofDatasource("wiki")
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(3, Granularities.DAY)
                           .startingAt("2013-01-01")
                           .eachOfSizeInMb(400);
@@ -67,7 +68,7 @@ public class ActionBasedPublishedSegmentRetrieverTest
     EasyMock.expect(
         taskActionClient.submit(
             new RetrieveSegmentsByIdAction(
-                "wiki",
+                TestDataSource.WIKI,
                 segments.stream().map(segment -> segment.getId().toString()).collect(Collectors.toSet())
             )
         )
@@ -89,20 +90,20 @@ public class ActionBasedPublishedSegmentRetrieverTest
   public void testRetrieveUsedSegmentsIfNotFoundById() throws IOException
   {
     final List<DataSegment> segments =
-        CreateDataSegments.ofDatasource("wiki")
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(3, Granularities.DAY)
                           .startingAt("2013-01-01")
                           .eachOfSizeInMb(400);
 
     EasyMock.expect(
         taskActionClient.submit(
-            new RetrieveSegmentsByIdAction("wiki", EasyMock.anyObject())
+            new RetrieveSegmentsByIdAction(TestDataSource.WIKI, EasyMock.anyObject())
         )
     ).andThrow(InvalidInput.exception("task action not supported yet")).once();
     EasyMock.expect(
         taskActionClient.submit(
             new RetrieveUsedSegmentsAction(
-                "wiki",
+                TestDataSource.WIKI,
                 Collections.singletonList(Intervals.of("2013-01-01/P3D")),
                 Segments.INCLUDING_OVERSHADOWED
             )
@@ -128,8 +129,8 @@ public class ActionBasedPublishedSegmentRetrieverTest
         DruidException.class,
         () -> segmentRetriever.findPublishedSegments(
             ImmutableSet.of(
-                SegmentId.of("wiki", Intervals.ETERNITY, "v1", 0),
-                SegmentId.of("koala", Intervals.ETERNITY, "v1", 0)
+                SegmentId.of(TestDataSource.WIKI, Intervals.ETERNITY, "v1", 0),
+                SegmentId.of(TestDataSource.KOALA, Intervals.ETERNITY, "v1", 0)
             )
         )
     );
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueueTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueueTest.java
index a71cacca13a..6a5c84082f2 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueueTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocationQueueTest.java
@@ -27,6 +27,7 @@ import org.apache.druid.indexing.overlord.config.TaskLockConfig;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.metrics.StubServiceEmitter;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
 import org.apache.druid.server.coordinator.simulate.BlockingExecutorService;
 import org.apache.druid.server.coordinator.simulate.WrappingScheduledExecutorService;
@@ -48,9 +49,6 @@ public class SegmentAllocationQueueTest
   @Rule
   public TaskActionTestKit taskActionTestKit = new TaskActionTestKit();
 
-  private static final String DS_WIKI = "wiki";
-  private static final String DS_KOALA = "koala";
-
   private SegmentAllocationQueue allocationQueue;
 
   private StubServiceEmitter emitter;
@@ -105,14 +103,14 @@ public class SegmentAllocationQueueTest
   public void testBatchWithMultipleTimestamps()
   {
     verifyAllocationWithBatching(
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .forTimestamp("2022-01-01T01:00:00")
                          .withSegmentGranularity(Granularities.DAY)
                          .withQueryGranularity(Granularities.SECOND)
                          .withLockGranularity(LockGranularity.TIME_CHUNK)
                          .withSequenceName("seq_1")
                          .build(),
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .forTimestamp("2022-01-01T02:00:00")
                          .withSegmentGranularity(Granularities.DAY)
                          .withQueryGranularity(Granularities.SECOND)
@@ -127,9 +125,9 @@ public class SegmentAllocationQueueTest
   public void testBatchWithExclusiveLocks()
   {
     verifyAllocationWithBatching(
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withTaskLockType(TaskLockType.EXCLUSIVE).build(),
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withTaskLockType(TaskLockType.EXCLUSIVE).build(),
         true
     );
@@ -139,9 +137,9 @@ public class SegmentAllocationQueueTest
   public void testBatchWithSharedLocks()
   {
     verifyAllocationWithBatching(
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withTaskLockType(TaskLockType.SHARED).build(),
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withTaskLockType(TaskLockType.SHARED).build(),
         true
     );
@@ -151,9 +149,9 @@ public class SegmentAllocationQueueTest
   public void testBatchWithMultipleQueryGranularities()
   {
     verifyAllocationWithBatching(
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withQueryGranularity(Granularities.SECOND).build(),
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withQueryGranularity(Granularities.MINUTE).build(),
         true
     );
@@ -163,8 +161,8 @@ public class SegmentAllocationQueueTest
   public void testMultipleDatasourcesCannotBatch()
   {
     verifyAllocationWithBatching(
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1")).build(),
-        allocateRequest().forTask(createTask(DS_KOALA, "group_1")).build(),
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1")).build(),
+        allocateRequest().forTask(createTask(TestDataSource.KOALA, "group_1")).build(),
         false
     );
   }
@@ -173,8 +171,8 @@ public class SegmentAllocationQueueTest
   public void testMultipleGroupIdsCannotBatch()
   {
     verifyAllocationWithBatching(
-        allocateRequest().forTask(createTask(DS_WIKI, "group_2")).build(),
-        allocateRequest().forTask(createTask(DS_WIKI, "group_3")).build(),
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_2")).build(),
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_3")).build(),
         false
     );
   }
@@ -183,9 +181,9 @@ public class SegmentAllocationQueueTest
   public void testMultipleLockGranularitiesCannotBatch()
   {
     verifyAllocationWithBatching(
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withLockGranularity(LockGranularity.TIME_CHUNK).build(),
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withLockGranularity(LockGranularity.SEGMENT).build(),
         false
     );
@@ -195,10 +193,10 @@ public class SegmentAllocationQueueTest
   public void testMultipleAllocateIntervalsCannotBatch()
   {
     verifyAllocationWithBatching(
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .forTimestamp("2022-01-01")
                          .withSegmentGranularity(Granularities.DAY).build(),
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .forTimestamp("2022-01-02")
                          .withSegmentGranularity(Granularities.DAY).build(),
         false
@@ -209,13 +207,13 @@ public class SegmentAllocationQueueTest
   public void testConflictingPendingSegment()
   {
     SegmentAllocateRequest hourSegmentRequest =
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withSegmentGranularity(Granularities.HOUR)
                          .build();
     Future<SegmentIdWithShardSpec> hourSegmentFuture = allocationQueue.add(hourSegmentRequest);
 
     SegmentAllocateRequest halfHourSegmentRequest =
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1"))
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1"))
                          .withSegmentGranularity(Granularities.THIRTY_MINUTE)
                          .build();
     Future<SegmentIdWithShardSpec> halfHourSegmentFuture = allocationQueue.add(halfHourSegmentRequest);
@@ -231,12 +229,12 @@ public class SegmentAllocationQueueTest
   {
     for (int i = 0; i < 2000; ++i) {
       SegmentAllocateRequest request =
-          allocateRequest().forTask(createTask(DS_WIKI, "group_" + i)).build();
+          allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_" + i)).build();
       allocationQueue.add(request);
     }
 
     SegmentAllocateRequest request =
-        allocateRequest().forTask(createTask(DS_WIKI, "next_group")).build();
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "next_group")).build();
     Future<SegmentIdWithShardSpec> future = allocationQueue.add(request);
 
     // Verify that the future is already complete and segment allocation has failed
@@ -253,14 +251,14 @@ public class SegmentAllocationQueueTest
   {
     for (int i = 0; i < 500; ++i) {
       SegmentAllocateRequest request =
-          allocateRequest().forTask(createTask(DS_WIKI, "group_1")).build();
+          allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1")).build();
       allocationQueue.add(request);
     }
 
     // Verify that next request is added to a new batch
     Assert.assertEquals(1, allocationQueue.size());
     SegmentAllocateRequest request =
-        allocateRequest().forTask(createTask(DS_WIKI, "group_1")).build();
+        allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_1")).build();
     allocationQueue.add(request);
     Assert.assertEquals(2, allocationQueue.size());
   }
@@ -271,7 +269,7 @@ public class SegmentAllocationQueueTest
     final List<Future<SegmentIdWithShardSpec>> segmentFutures = new ArrayList<>();
     for (int i = 0; i < 10; ++i) {
       SegmentAllocateRequest request =
-          allocateRequest().forTask(createTask(DS_WIKI, "group_" + i))
+          allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_" + i))
                            .withSequenceName("sequence_1")
                            .withPreviousSegmentId("segment_1")
                            .build();
@@ -299,7 +297,7 @@ public class SegmentAllocationQueueTest
     final List<Future<SegmentIdWithShardSpec>> segmentFutures = new ArrayList<>();
     for (int i = 0; i < 10; ++i) {
       SegmentAllocateRequest request =
-          allocateRequest().forTask(createTask(DS_WIKI, "group_" + i)).build();
+          allocateRequest().forTask(createTask(TestDataSource.WIKI, "group_" + i)).build();
       segmentFutures.add(allocationQueue.add(request));
     }
 
@@ -348,7 +346,7 @@ public class SegmentAllocationQueueTest
   private SegmentAllocateActionBuilder allocateRequest()
   {
     return new SegmentAllocateActionBuilder()
-        .forDatasource(DS_WIKI)
+        .forDatasource(TestDataSource.WIKI)
         .forTimestamp("2022-01-01")
         .withLockGranularity(LockGranularity.TIME_CHUNK)
         .withTaskLockType(TaskLockType.SHARED)
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PartialCompactionTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PartialCompactionTest.java
index 15201e884d3..9962dfcd821 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PartialCompactionTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PartialCompactionTest.java
@@ -29,8 +29,6 @@ import org.apache.druid.indexer.partitions.HashedPartitionsSpec;
 import org.apache.druid.indexer.partitions.PartitionsSpec;
 import org.apache.druid.indexer.partitions.SingleDimensionPartitionsSpec;
 import org.apache.druid.indexing.common.LockGranularity;
-import org.apache.druid.indexing.common.RetryPolicyConfig;
-import org.apache.druid.indexing.common.RetryPolicyFactory;
 import org.apache.druid.indexing.common.task.CompactionTask;
 import org.apache.druid.indexing.common.task.CompactionTask.Builder;
 import org.apache.druid.indexing.common.task.SpecificSegmentsSpec;
@@ -69,7 +67,6 @@ public class PartialCompactionTest extends AbstractMultiPhaseParallelIndexingTes
       0
   );
   private static final Interval INTERVAL_TO_INDEX = Intervals.of("2017-12/P1M");
-  private static final RetryPolicyFactory RETRY_POLICY_FACTORY = new RetryPolicyFactory(new RetryPolicyConfig());
 
   private File inputDir;
 
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndAppendTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndAppendTest.java
index f179e4707ec..96c72e7130d 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndAppendTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndAppendTest.java
@@ -53,6 +53,7 @@ import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.segment.IndexIO;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.segment.column.ColumnConfig;
 import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
 import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
@@ -105,8 +106,6 @@ public class ConcurrentReplaceAndAppendTest extends IngestionTestBase
   private static final Interval OCT_NOV_DEC_23 = Intervals.of("2023-10-01/2024-01-01");
   private static final Interval FIRST_OF_JAN_23 = Intervals.of("2023-01-01/2023-01-02");
 
-  private static final String WIKI = "wiki";
-
   private TaskQueue taskQueue;
   private TaskActionClientFactory taskActionClientFactory;
   private TaskActionClient dummyTaskActionClient;
@@ -983,7 +982,7 @@ public class ConcurrentReplaceAndAppendTest extends IngestionTestBase
 
       Collection<DataSegment> allUsedSegments = dummyTaskActionClient.submit(
           new RetrieveUsedSegmentsAction(
-              WIKI,
+              TestDataSource.WIKI,
               ImmutableList.of(interval),
               visibility
           )
@@ -1001,7 +1000,7 @@ public class ConcurrentReplaceAndAppendTest extends IngestionTestBase
       final TaskActionClient taskActionClient = taskActionClientFactory.create(task);
       Collection<DataSegment> allUsedSegments = taskActionClient.submit(
           new RetrieveUsedSegmentsAction(
-              WIKI,
+              TestDataSource.WIKI,
               Collections.singletonList(interval)
           )
       );
@@ -1037,7 +1036,7 @@ public class ConcurrentReplaceAndAppendTest extends IngestionTestBase
   private DataSegment createSegment(Interval interval, String version)
   {
     return DataSegment.builder()
-                      .dataSource(WIKI)
+                      .dataSource(TestDataSource.WIKI)
                       .interval(interval)
                       .version(version)
                       .size(100)
@@ -1046,7 +1045,7 @@ public class ConcurrentReplaceAndAppendTest extends IngestionTestBase
 
   private ActionsTestTask createAndStartTask()
   {
-    ActionsTestTask task = new ActionsTestTask(WIKI, "test_" + groupId.incrementAndGet(), taskActionClientFactory);
+    ActionsTestTask task = new ActionsTestTask(TestDataSource.WIKI, "test_" + groupId.incrementAndGet(), taskActionClientFactory);
     taskQueue.add(task);
     runningTasks.add(task);
     return task;
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndStreamingAppendTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndStreamingAppendTest.java
index a8ce15538d3..1b2fc1bbde0 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndStreamingAppendTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndStreamingAppendTest.java
@@ -57,6 +57,7 @@ import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.metadata.PendingSegmentRecord;
 import org.apache.druid.segment.IndexIO;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.segment.column.ColumnConfig;
 import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
 import org.apache.druid.server.DruidNode;
@@ -108,8 +109,6 @@ public class ConcurrentReplaceAndStreamingAppendTest extends IngestionTestBase
   private static final Interval JAN_23 = Intervals.of("2023-01/2023-02");
   private static final Interval FIRST_OF_JAN_23 = Intervals.of("2023-01-01/2023-01-02");
 
-  private static final String WIKI = "wiki";
-
   private TaskQueue taskQueue;
   private TaskActionClientFactory taskActionClientFactory;
   private TaskActionClient dummyTaskActionClient;
@@ -130,8 +129,8 @@ public class ConcurrentReplaceAndStreamingAppendTest extends IngestionTestBase
   public void setUpIngestionTestBase() throws IOException
   {
     EasyMock.reset(supervisorManager);
-    EasyMock.expect(supervisorManager.getActiveSupervisorIdForDatasourceWithAppendLock(WIKI))
-            .andReturn(Optional.of(WIKI)).anyTimes();
+    EasyMock.expect(supervisorManager.getActiveSupervisorIdForDatasourceWithAppendLock(TestDataSource.WIKI))
+            .andReturn(Optional.of(TestDataSource.WIKI)).anyTimes();
     super.setUpIngestionTestBase();
     final TaskConfig taskConfig = new TaskConfigBuilder().build();
     taskActionClientFactory = createActionClientFactory();
@@ -708,7 +707,7 @@ public class ConcurrentReplaceAndStreamingAppendTest extends IngestionTestBase
     try {
       Collection<DataSegment> allUsedSegments = dummyTaskActionClient.submit(
           new RetrieveUsedSegmentsAction(
-              WIKI,
+              TestDataSource.WIKI,
               ImmutableList.of(interval),
               visibility
           )
@@ -741,9 +740,9 @@ public class ConcurrentReplaceAndStreamingAppendTest extends IngestionTestBase
 
   private DataSegment createSegment(Interval interval, String version)
   {
-    SegmentId id = SegmentId.of(WIKI, interval, version, null);
+    SegmentId id = SegmentId.of(TestDataSource.WIKI, interval, version, null);
     return DataSegment.builder()
-                      .dataSource(WIKI)
+                      .dataSource(TestDataSource.WIKI)
                       .interval(interval)
                       .version(version)
                       .loadSpec(Collections.singletonMap(id.toString(), id.toString()))
@@ -753,7 +752,7 @@ public class ConcurrentReplaceAndStreamingAppendTest extends IngestionTestBase
 
   private ActionsTestTask createAndStartTask()
   {
-    ActionsTestTask task = new ActionsTestTask(WIKI, "test_" + groupId.incrementAndGet(), taskActionClientFactory);
+    ActionsTestTask task = new ActionsTestTask(TestDataSource.WIKI, "test_" + groupId.incrementAndGet(), taskActionClientFactory);
     taskQueue.add(task);
     runningTasks.add(task);
     return task;
@@ -827,7 +826,7 @@ public class ConcurrentReplaceAndStreamingAppendTest extends IngestionTestBase
     try {
       return dummyTaskActionClient.submit(
           new RetrieveUsedSegmentsAction(
-              WIKI,
+              TestDataSource.WIKI,
               ImmutableList.of(Intervals.ETERNITY),
               Segments.INCLUDING_OVERSHADOWED
           )
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/compact/CompactionSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CompactionSupervisorSpecTest.java
new file mode 100644
index 00000000000..2e6a1cf8cc3
--- /dev/null
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CompactionSupervisorSpecTest.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.compact;
+
+import com.fasterxml.jackson.databind.InjectableValues;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.guice.SupervisorModule;
+import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.server.coordinator.CompactionConfigValidationResult;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+
+public class CompactionSupervisorSpecTest
+{
+  private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper();
+  private CompactionScheduler scheduler;
+
+  @Before
+  public void setUp()
+  {
+    scheduler = Mockito.mock(CompactionScheduler.class);
+    Mockito.when(scheduler.validateCompactionConfig(ArgumentMatchers.any()))
+           .thenReturn(CompactionConfigValidationResult.success());
+
+    OBJECT_MAPPER.setInjectableValues(
+        new InjectableValues.Std()
+            .addValue(CompactionScheduler.class, scheduler)
+    );
+    OBJECT_MAPPER.registerModules(
+        new SupervisorModule().getJacksonModules()
+    );
+  }
+
+  @Test
+  public void testSerdeOfActiveSpec()
+  {
+    testSerde(
+        new CompactionSupervisorSpec(
+            DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
+            false,
+            scheduler
+        )
+    );
+  }
+
+  @Test
+  public void testSerdeOfSuspendedSpec()
+  {
+    testSerde(
+        new CompactionSupervisorSpec(
+            DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
+            true,
+            scheduler
+        )
+    );
+  }
+
+  @Test
+  public void testInvalidSpecThrowsException()
+  {
+    Mockito.when(scheduler.validateCompactionConfig(ArgumentMatchers.any()))
+           .thenReturn(CompactionConfigValidationResult.failure("bad spec"));
+    final DruidException exception = Assert.assertThrows(
+        DruidException.class,
+        () -> new CompactionSupervisorSpec(null, false, scheduler)
+    );
+    Assert.assertEquals(
+        "Compaction supervisor 'spec' is invalid. Reason[bad spec].",
+        exception.getMessage()
+    );
+  }
+
+  @Test
+  public void testGetIdAndDataSources()
+  {
+    final CompactionSupervisorSpec activeSpec = new CompactionSupervisorSpec(
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
+        false,
+        scheduler
+    );
+    Assert.assertEquals("autocompact__wiki", activeSpec.getId());
+    Assert.assertEquals(Collections.singletonList(TestDataSource.WIKI), activeSpec.getDataSources());
+    Assert.assertFalse(activeSpec.isSuspended());
+  }
+
+  @Test
+  public void testStartStopSupervisorForActiveSpec()
+  {
+    Mockito.when(scheduler.isRunning()).thenReturn(true);
+
+    final DataSourceCompactionConfig spec
+        = DataSourceCompactionConfig.builder()
+                                    .forDataSource(TestDataSource.WIKI)
+                                    .build();
+    final CompactionSupervisorSpec activeSpec
+        = new CompactionSupervisorSpec(spec, false, scheduler);
+
+    final CompactionSupervisor supervisor = activeSpec.createSupervisor();
+    Assert.assertEquals(CompactionSupervisor.State.RUNNING, supervisor.getState());
+
+    supervisor.start();
+    supervisor.stop(false);
+
+    Mockito.verify(scheduler, Mockito.times(1)).startCompaction(TestDataSource.WIKI, spec);
+    Mockito.verify(scheduler, Mockito.times(1)).stopCompaction(TestDataSource.WIKI);
+  }
+
+  @Test
+  public void testStartStopSupervisorWhenSchedulerStopped()
+  {
+    final DataSourceCompactionConfig spec
+        = DataSourceCompactionConfig.builder()
+                                    .forDataSource(TestDataSource.WIKI)
+                                    .build();
+    final CompactionSupervisorSpec activeSpec
+        = new CompactionSupervisorSpec(spec, false, scheduler);
+
+    final CompactionSupervisor supervisor = activeSpec.createSupervisor();
+    Assert.assertEquals(CompactionSupervisor.State.SCHEDULER_STOPPED, supervisor.getState());
+
+    supervisor.start();
+    supervisor.stop(false);
+
+    Mockito.verify(scheduler, Mockito.times(1)).startCompaction(TestDataSource.WIKI, spec);
+    Mockito.verify(scheduler, Mockito.times(1)).stopCompaction(TestDataSource.WIKI);
+  }
+
+  @Test
+  public void testStartStopSupervisorForSuspendedSpec()
+  {
+    Mockito.when(scheduler.isRunning()).thenReturn(true);
+
+    final DataSourceCompactionConfig spec
+        = DataSourceCompactionConfig.builder()
+                                    .forDataSource(TestDataSource.WIKI)
+                                    .build();
+    final CompactionSupervisorSpec suspendedSpec
+        = new CompactionSupervisorSpec(spec, true, scheduler);
+
+    final CompactionSupervisor supervisor = suspendedSpec.createSupervisor();
+    Assert.assertEquals(CompactionSupervisor.State.SUSPENDED, supervisor.getState());
+
+    supervisor.start();
+    supervisor.stop(false);
+
+    Mockito.verify(scheduler, Mockito.times(2)).stopCompaction(TestDataSource.WIKI);
+  }
+
+  @Test
+  public void testCreateSuspendedSpec()
+  {
+    final CompactionSupervisorSpec activeSpec = new CompactionSupervisorSpec(
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
+        false,
+        scheduler
+    );
+    Assert.assertFalse(activeSpec.isSuspended());
+
+    final CompactionSupervisorSpec suspendedSpec = activeSpec.createSuspendedSpec();
+    Assert.assertTrue(suspendedSpec.isSuspended());
+    Assert.assertEquals(activeSpec.getId(), suspendedSpec.getId());
+    Assert.assertEquals(activeSpec.getSpec(), suspendedSpec.getSpec());
+    Assert.assertEquals(activeSpec.getDataSources(), suspendedSpec.getDataSources());
+  }
+
+  @Test
+  public void testCreateRunningSpec()
+  {
+    final CompactionSupervisorSpec suspendedSpec = new CompactionSupervisorSpec(
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
+        true,
+        scheduler
+    );
+    Assert.assertTrue(suspendedSpec.isSuspended());
+
+    final CompactionSupervisorSpec activeSpec = suspendedSpec.createRunningSpec();
+    Assert.assertFalse(activeSpec.isSuspended());
+    Assert.assertEquals(activeSpec.getId(), suspendedSpec.getId());
+    Assert.assertEquals(activeSpec.getSpec(), suspendedSpec.getSpec());
+    Assert.assertEquals(activeSpec.getDataSources(), suspendedSpec.getDataSources());
+  }
+
+  private void testSerde(CompactionSupervisorSpec spec)
+  {
+    try {
+      String json = OBJECT_MAPPER.writeValueAsString(spec);
+      SupervisorSpec deserialized = OBJECT_MAPPER.readValue(json, SupervisorSpec.class);
+      Assert.assertTrue(deserialized instanceof CompactionSupervisorSpec);
+
+      final CompactionSupervisorSpec observedSpec = (CompactionSupervisorSpec) deserialized;
+      Assert.assertEquals(spec.isSuspended(), observedSpec.isSuspended());
+      Assert.assertEquals(spec.getSpec(), observedSpec.getSpec());
+      Assert.assertEquals(spec.getId(), observedSpec.getId());
+      Assert.assertEquals(spec.getDataSources(), observedSpec.getDataSources());
+    }
+    catch (Exception e) {
+      throw DruidException.defensive(e, "Error while performing serde");
+    }
+  }
+}
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java
new file mode 100644
index 00000000000..3133e2b9466
--- /dev/null
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.compact;
+
+import com.fasterxml.jackson.databind.InjectableValues;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.druid.client.indexing.ClientMSQContext;
+import org.apache.druid.guice.IndexingServiceTuningConfigModule;
+import org.apache.druid.indexer.CompactionEngine;
+import org.apache.druid.indexing.common.SegmentCacheManagerFactory;
+import org.apache.druid.indexing.common.config.TaskStorageConfig;
+import org.apache.druid.indexing.common.task.CompactionTask;
+import org.apache.druid.indexing.common.task.Task;
+import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage;
+import org.apache.druid.indexing.overlord.TaskLockbox;
+import org.apache.druid.indexing.overlord.TaskMaster;
+import org.apache.druid.indexing.overlord.TaskQueryTool;
+import org.apache.druid.indexing.overlord.TaskQueue;
+import org.apache.druid.indexing.overlord.TaskRunner;
+import org.apache.druid.indexing.overlord.setup.DefaultWorkerBehaviorConfig;
+import org.apache.druid.indexing.overlord.setup.WorkerBehaviorConfig;
+import org.apache.druid.indexing.test.TestIndexerMetadataStorageCoordinator;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.metrics.StubServiceEmitter;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.segment.TestIndex;
+import org.apache.druid.server.compaction.CompactionSimulateResult;
+import org.apache.druid.server.compaction.CompactionStatistics;
+import org.apache.druid.server.compaction.CompactionStatus;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
+import org.apache.druid.server.compaction.Table;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.coordinator.ClusterCompactionConfig;
+import org.apache.druid.server.coordinator.CompactionConfigValidationResult;
+import org.apache.druid.server.coordinator.CompactionSupervisorConfig;
+import org.apache.druid.server.coordinator.CoordinatorOverlordServiceConfig;
+import org.apache.druid.server.coordinator.CreateDataSegments;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+import org.apache.druid.server.coordinator.DruidCompactionConfig;
+import org.apache.druid.server.coordinator.simulate.BlockingExecutorService;
+import org.apache.druid.server.coordinator.simulate.TestSegmentsMetadataManager;
+import org.apache.druid.server.coordinator.simulate.WrappingScheduledExecutorService;
+import org.apache.druid.server.coordinator.stats.Stats;
+import org.apache.druid.timeline.DataSegment;
+import org.joda.time.Period;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class OverlordCompactionSchedulerTest
+{
+  private static final ObjectMapper OBJECT_MAPPER;
+
+  static {
+    OBJECT_MAPPER = new DefaultObjectMapper();
+    OBJECT_MAPPER.registerModules(new IndexingServiceTuningConfigModule().getJacksonModules());
+    OBJECT_MAPPER.setInjectableValues(
+        new InjectableValues
+            .Std()
+            .addValue(
+                SegmentCacheManagerFactory.class,
+                new SegmentCacheManagerFactory(TestIndex.INDEX_IO, OBJECT_MAPPER)
+            )
+    );
+  }
+
+  private CompactionSupervisorConfig supervisorConfig;
+  private DruidCompactionConfig compactionConfig;
+  private CoordinatorOverlordServiceConfig coordinatorOverlordServiceConfig;
+
+  private TaskMaster taskMaster;
+  private TaskRunner taskRunner;
+  private TaskQueue taskQueue;
+  private BlockingExecutorService executor;
+
+  private HeapMemoryTaskStorage taskStorage;
+  private TestSegmentsMetadataManager segmentsMetadataManager;
+  private StubServiceEmitter serviceEmitter;
+
+  private OverlordCompactionScheduler scheduler;
+
+  @Before
+  public void setUp()
+  {
+    taskRunner = Mockito.mock(TaskRunner.class);
+    taskQueue = Mockito.mock(TaskQueue.class);
+
+    taskMaster = new TaskMaster(null, null);
+    taskMaster.becomeLeader(taskRunner, taskQueue);
+
+    taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null));
+
+    executor = new BlockingExecutorService("test");
+    serviceEmitter = new StubServiceEmitter();
+    segmentsMetadataManager = new TestSegmentsMetadataManager();
+
+    supervisorConfig = new CompactionSupervisorConfig(true);
+    compactionConfig = DruidCompactionConfig.empty();
+    coordinatorOverlordServiceConfig = new CoordinatorOverlordServiceConfig(false, null);
+
+    initScheduler();
+  }
+
+  private void initScheduler()
+  {
+    TaskLockbox taskLockbox = new TaskLockbox(taskStorage, new TestIndexerMetadataStorageCoordinator());
+    WorkerBehaviorConfig defaultWorkerConfig
+        = new DefaultWorkerBehaviorConfig(WorkerBehaviorConfig.DEFAULT_STRATEGY, null);
+    scheduler = new OverlordCompactionScheduler(
+        taskMaster,
+        new TaskQueryTool(taskStorage, taskLockbox, taskMaster, null, () -> defaultWorkerConfig),
+        segmentsMetadataManager,
+        () -> compactionConfig,
+        new CompactionStatusTracker(OBJECT_MAPPER),
+        supervisorConfig,
+        coordinatorOverlordServiceConfig,
+        (nameFormat, numThreads) -> new WrappingScheduledExecutorService("test", executor, false),
+        serviceEmitter,
+        OBJECT_MAPPER
+    );
+  }
+
+  @Test
+  public void testStartStopWhenSchedulerIsEnabled()
+  {
+    supervisorConfig = new CompactionSupervisorConfig(true);
+    Assert.assertFalse(scheduler.isRunning());
+
+    scheduler.start();
+    Assert.assertTrue(scheduler.isRunning());
+    Assert.assertTrue(executor.hasPendingTasks());
+    scheduler.stop();
+    Assert.assertFalse(scheduler.isRunning());
+    Assert.assertTrue(executor.hasPendingTasks());
+
+    scheduler.start();
+    Assert.assertTrue(scheduler.isRunning());
+    scheduler.stop();
+    Assert.assertFalse(scheduler.isRunning());
+  }
+
+  @Test
+  public void testStartStopWhenScheduledIsDisabled()
+  {
+    supervisorConfig = new CompactionSupervisorConfig(false);
+    initScheduler();
+
+    Assert.assertFalse(scheduler.isRunning());
+    scheduler.start();
+    Assert.assertFalse(scheduler.isRunning());
+    Assert.assertFalse(executor.hasPendingTasks());
+    scheduler.stop();
+    Assert.assertFalse(scheduler.isRunning());
+    Assert.assertFalse(executor.hasPendingTasks());
+  }
+
+  @Test
+  public void testSegmentsAreNotPolledWhenSchedulerIsDisabled()
+  {
+    supervisorConfig = new CompactionSupervisorConfig(false);
+    initScheduler();
+
+    verifySegmentPolling(false);
+  }
+
+  @Test
+  public void testSegmentsArePolledWhenRunningInStandaloneMode()
+  {
+    coordinatorOverlordServiceConfig = new CoordinatorOverlordServiceConfig(false, null);
+    initScheduler();
+
+    verifySegmentPolling(true);
+  }
+
+  @Test
+  public void testSegmentsAreNotPolledWhenRunningInCoordinatorMode()
+  {
+    coordinatorOverlordServiceConfig = new CoordinatorOverlordServiceConfig(true, "overlord");
+    initScheduler();
+
+    verifySegmentPolling(false);
+  }
+
+  private void verifySegmentPolling(boolean enabled)
+  {
+    scheduler.start();
+    Assert.assertEquals(enabled, segmentsMetadataManager.isPollingDatabasePeriodically());
+
+    scheduler.stop();
+    Assert.assertFalse(segmentsMetadataManager.isPollingDatabasePeriodically());
+  }
+
+  @Test
+  public void testNullCompactionConfigIsInvalid()
+  {
+    final CompactionConfigValidationResult result = scheduler.validateCompactionConfig(null);
+    Assert.assertFalse(result.isValid());
+    Assert.assertEquals("Cannot be null", result.getReason());
+  }
+
+  @Test
+  public void testMsqCompactionConfigWithOneMaxTasksIsInvalid()
+  {
+    final DataSourceCompactionConfig datasourceConfig = DataSourceCompactionConfig
+        .builder()
+        .forDataSource(TestDataSource.WIKI)
+        .withEngine(CompactionEngine.MSQ)
+        .withTaskContext(Collections.singletonMap(ClientMSQContext.CTX_MAX_NUM_TASKS, 1))
+        .build();
+
+    final CompactionConfigValidationResult result = scheduler.validateCompactionConfig(datasourceConfig);
+    Assert.assertFalse(result.isValid());
+    Assert.assertEquals(
+        "MSQ: Context maxNumTasks[1] must be at least 2 (1 controller + 1 worker)",
+        result.getReason()
+    );
+  }
+
+  @Test
+  public void testStartCompactionForDatasource()
+  {
+    final List<DataSegment> wikiSegments = CreateDataSegments.ofDatasource(TestDataSource.WIKI).eachOfSizeInMb(100);
+    wikiSegments.forEach(segmentsMetadataManager::addSegment);
+
+    scheduler.start();
+    scheduler.startCompaction(
+        TestDataSource.WIKI,
+        DataSourceCompactionConfig.builder()
+                                  .forDataSource(TestDataSource.WIKI)
+                                  .withSkipOffsetFromLatest(Period.seconds(0))
+                                  .build()
+    );
+
+    executor.finishNextPendingTask();
+
+    ArgumentCaptor<Task> taskArgumentCaptor = ArgumentCaptor.forClass(Task.class);
+    Mockito.verify(taskQueue, Mockito.times(1)).add(taskArgumentCaptor.capture());
+
+    Task submittedTask = taskArgumentCaptor.getValue();
+    Assert.assertNotNull(submittedTask);
+    Assert.assertTrue(submittedTask instanceof CompactionTask);
+
+    final CompactionTask compactionTask = (CompactionTask) submittedTask;
+    Assert.assertEquals(TestDataSource.WIKI, compactionTask.getDataSource());
+
+    final AutoCompactionSnapshot.Builder expectedSnapshot = AutoCompactionSnapshot.builder(TestDataSource.WIKI);
+    expectedSnapshot.incrementCompactedStats(CompactionStatistics.create(100_000_000, 1, 1));
+
+    Assert.assertEquals(
+        expectedSnapshot.build(),
+        scheduler.getCompactionSnapshot(TestDataSource.WIKI)
+    );
+    Assert.assertEquals(
+        Collections.singletonMap(TestDataSource.WIKI, expectedSnapshot.build()),
+        scheduler.getAllCompactionSnapshots()
+    );
+
+    serviceEmitter.verifyValue(Stats.Compaction.SUBMITTED_TASKS.getMetricName(), 1L);
+    serviceEmitter.verifyValue(Stats.Compaction.COMPACTED_BYTES.getMetricName(), 100_000_000L);
+
+    scheduler.stop();
+  }
+
+  @Test
+  public void testStopCompactionForDatasource()
+  {
+    final List<DataSegment> wikiSegments = CreateDataSegments.ofDatasource(TestDataSource.WIKI).eachOfSizeInMb(100);
+    wikiSegments.forEach(segmentsMetadataManager::addSegment);
+
+    scheduler.start();
+    scheduler.startCompaction(
+        TestDataSource.WIKI,
+        DataSourceCompactionConfig.builder()
+                                  .forDataSource(TestDataSource.WIKI)
+                                  .withSkipOffsetFromLatest(Period.seconds(0))
+                                  .build()
+    );
+    scheduler.stopCompaction(TestDataSource.WIKI);
+
+    executor.finishNextPendingTask();
+
+    Mockito.verify(taskQueue, Mockito.never()).add(ArgumentMatchers.any());
+
+    Assert.assertNull(scheduler.getCompactionSnapshot(TestDataSource.WIKI));
+    Assert.assertTrue(scheduler.getAllCompactionSnapshots().isEmpty());
+
+    serviceEmitter.verifyNotEmitted(Stats.Compaction.SUBMITTED_TASKS.getMetricName());
+    serviceEmitter.verifyNotEmitted(Stats.Compaction.COMPACTED_BYTES.getMetricName());
+
+    scheduler.stop();
+  }
+
+  @Test
+  public void testRunSimulation()
+  {
+    final List<DataSegment> wikiSegments = CreateDataSegments
+        .ofDatasource(TestDataSource.WIKI)
+        .forIntervals(1, Granularities.DAY)
+        .startingAt("2013-01-01")
+        .withNumPartitions(10)
+        .eachOfSizeInMb(100);
+    wikiSegments.forEach(segmentsMetadataManager::addSegment);
+
+    scheduler.start();
+    scheduler.startCompaction(
+        TestDataSource.WIKI,
+        DataSourceCompactionConfig.builder()
+                                  .forDataSource(TestDataSource.WIKI)
+                                  .withSkipOffsetFromLatest(Period.seconds(0))
+                                  .build()
+    );
+
+    final CompactionSimulateResult simulateResult = scheduler.simulateRunWithConfigUpdate(
+        new ClusterCompactionConfig(null, null, null, null, null)
+    );
+    Assert.assertEquals(1, simulateResult.getCompactionStates().size());
+    final Table pendingCompactionTable = simulateResult.getCompactionStates().get(CompactionStatus.State.PENDING);
+    Assert.assertEquals(
+        Arrays.asList("dataSource", "interval", "numSegments", "bytes", "maxTaskSlots", "reasonToCompact"),
+        pendingCompactionTable.getColumnNames()
+    );
+    Assert.assertEquals(
+        Collections.singletonList(
+            Arrays.asList(
+                TestDataSource.WIKI,
+                Intervals.of("2013-01-01/P1D"),
+                10,
+                1_000_000_000L,
+                1,
+                "not compacted yet"
+            )
+        ),
+        pendingCompactionTable.getRows()
+    );
+
+    scheduler.stopCompaction(TestDataSource.WIKI);
+
+    final CompactionSimulateResult simulateResultWhenDisabled = scheduler.simulateRunWithConfigUpdate(
+        new ClusterCompactionConfig(null, null, null, null, null)
+    );
+    Assert.assertTrue(simulateResultWhenDisabled.getCompactionStates().isEmpty());
+
+    scheduler.stop();
+  }
+
+}
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java
index 93f80ac9709..c13a1571765 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java
@@ -145,7 +145,7 @@ public class OverlordResourceTest
         taskLockbox,
         taskMaster,
         provisioningStrategy,
-        configManager
+        () -> configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class).get()
     );
     indexerMetadataStorageAdapter = EasyMock.createStrictMock(IndexerMetadataStorageAdapter.class);
     req = EasyMock.createStrictMock(HttpServletRequest.class);
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java
index d1c2167b929..f1a8b65a509 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java
@@ -47,6 +47,7 @@ import org.apache.druid.indexing.common.config.TaskStorageConfig;
 import org.apache.druid.indexing.common.task.NoopTask;
 import org.apache.druid.indexing.common.task.NoopTaskContextEnricher;
 import org.apache.druid.indexing.common.task.Task;
+import org.apache.druid.indexing.compact.CompactionScheduler;
 import org.apache.druid.indexing.overlord.DruidOverlord;
 import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage;
 import org.apache.druid.indexing.overlord.IndexerMetadataStorageAdapter;
@@ -257,6 +258,7 @@ public class OverlordTest
         EasyMock.createNiceMock(OverlordDutyExecutor.class),
         new TestDruidLeaderSelector(),
         EasyMock.createNiceMock(SegmentAllocationQueue.class),
+        EasyMock.createNiceMock(CompactionScheduler.class),
         new DefaultObjectMapper(),
         new NoopTaskContextEnricher()
     );
diff --git a/processing/src/main/java/org/apache/druid/java/util/common/Intervals.java b/processing/src/main/java/org/apache/druid/java/util/common/Intervals.java
index 4dfe09144e6..623f546349f 100644
--- a/processing/src/main/java/org/apache/druid/java/util/common/Intervals.java
+++ b/processing/src/main/java/org/apache/druid/java/util/common/Intervals.java
@@ -53,11 +53,6 @@ public final class Intervals
     return of(StringUtils.format(format, formatArgs));
   }
 
-  public static boolean isEmpty(Interval interval)
-  {
-    return interval.getStart().equals(interval.getEnd());
-  }
-
   /**
    * Returns true if the provided interval has endpoints that can be compared against other DateTimes using their
    * string representations.
diff --git a/processing/src/main/java/org/apache/druid/java/util/common/guava/Comparators.java b/processing/src/main/java/org/apache/druid/java/util/common/guava/Comparators.java
index 61e1912d2eb..9eb4578820b 100644
--- a/processing/src/main/java/org/apache/druid/java/util/common/guava/Comparators.java
+++ b/processing/src/main/java/org/apache/druid/java/util/common/guava/Comparators.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.java.util.common.guava;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Ordering;
 import org.joda.time.DateTimeComparator;
 import org.joda.time.Interval;
@@ -51,6 +52,23 @@ public class Comparators
     return (Ordering<T>) ALWAYS_EQUAL;
   }
 
+  /**
+   * Creates an ordering which always gives priority to the specified value.
+   * Other values are considered equal to each other.
+   */
+  public static <T> Ordering<T> alwaysFirst(T value)
+  {
+    Preconditions.checkNotNull(value, "value cannot be null");
+
+    return Ordering.from((o1, o2) -> {
+      if (value.equals(o1)) {
+        return value.equals(o2) ? 0 : -1;
+      } else {
+        return value.equals(o2) ? 1 : 0;
+      }
+    });
+  }
+
   @SuppressWarnings("unchecked")
   public static <T extends Comparable<? super T>> Ordering<T> naturalNullsFirst()
   {
diff --git a/processing/src/test/java/org/apache/druid/java/util/common/guava/ComparatorsTest.java b/processing/src/test/java/org/apache/druid/java/util/common/guava/ComparatorsTest.java
index 816e2d5a32a..53530852653 100644
--- a/processing/src/test/java/org/apache/druid/java/util/common/guava/ComparatorsTest.java
+++ b/processing/src/test/java/org/apache/druid/java/util/common/guava/ComparatorsTest.java
@@ -102,4 +102,23 @@ public class ComparatorsTest
         intervals
     );
   }
+
+  @Test
+  public void testAlwaysFirst()
+  {
+    final Comparator<Integer> comparator = Comparators.alwaysFirst(25);
+    Assert.assertEquals(0, comparator.compare(25, 25));
+    Assert.assertEquals(-1, comparator.compare(25, 1000));
+    Assert.assertEquals(1, comparator.compare(1000, 25));
+    Assert.assertEquals(-1, comparator.compare(25, 1));
+    Assert.assertEquals(1, comparator.compare(1, 25));
+    Assert.assertEquals(0, comparator.compare(10, 12));
+
+    Integer[] array = {25, 0, 1, 25, -10, 100, 31, 77, 89, 25};
+    Arrays.sort(array, comparator);
+    Assert.assertArrayEquals(
+        new Integer[]{25, 25, 25, 0, 1, -10, 100, 31, 77, 89},
+        array
+    );
+  }
 }
diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorModule.java b/processing/src/test/java/org/apache/druid/segment/TestDataSource.java
similarity index 68%
rename from server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorModule.java
rename to processing/src/test/java/org/apache/druid/segment/TestDataSource.java
index d281f55e52c..8789d5f3f9a 100644
--- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorModule.java
+++ b/processing/src/test/java/org/apache/druid/segment/TestDataSource.java
@@ -17,18 +17,13 @@
  * under the License.
  */
 
+package org.apache.druid.segment;
 
-package org.apache.druid.indexing.overlord.supervisor;
-
-import com.google.inject.Binder;
-import com.google.inject.Module;
-import org.apache.druid.guice.JsonConfigProvider;
-
-public class SupervisorModule implements Module
+/**
+ * DataSource names used in tests.
+ */
+public final class TestDataSource
 {
-  @Override
-  public void configure(Binder binder)
-  {
-    JsonConfigProvider.bind(binder, "druid.supervisor", SupervisorStateManagerConfig.class);
-  }
+  public static final String WIKI = "wiki";
+  public static final String KOALA = "koala";
 }
diff --git a/processing/src/test/java/org/apache/druid/timeline/SegmentTimelineTest.java b/processing/src/test/java/org/apache/druid/timeline/SegmentTimelineTest.java
index 0d765f24501..64f0ee776d2 100644
--- a/processing/src/test/java/org/apache/druid/timeline/SegmentTimelineTest.java
+++ b/processing/src/test/java/org/apache/druid/timeline/SegmentTimelineTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.timeline;
 
 import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.timeline.partition.NumberedShardSpec;
 import org.junit.Assert;
 import org.junit.Test;
@@ -67,7 +68,7 @@ public class SegmentTimelineTest
   private DataSegment createSegment(String interval, String version, int partitionNum, int totalNumPartitions)
   {
     return new DataSegment(
-        "wiki",
+        TestDataSource.WIKI,
         Intervals.of(interval),
         version,
         Collections.emptyMap(),
diff --git a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java
index 1bf942df9f8..330822067ad 100644
--- a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java
+++ b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java
@@ -19,7 +19,6 @@
 
 package org.apache.druid.metadata;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import org.apache.druid.client.DataSourcesSnapshot;
 import org.apache.druid.client.ImmutableDruidDataSource;
@@ -207,9 +206,6 @@ public interface SegmentsMetadataManager
       DateTime maxUsedStatusLastUpdatedTime
   );
 
-  @VisibleForTesting
-  void poll();
-
   /**
    * Populates used_status_last_updated column in the segments table iteratively until there are no segments with a NULL
    * value for that column.
diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
index 2c81603e529..6deed9c3f2b 100644
--- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
+++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
@@ -1008,8 +1008,8 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
     );
   }
 
-  @Override
-  public void poll()
+  @VisibleForTesting
+  void poll()
   {
     // See the comment to the pollLock field, explaining this synchronized block
     synchronized (pollLock) {
diff --git a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClient.java b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClient.java
index 310684206d2..6797a00b432 100644
--- a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClient.java
+++ b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClient.java
@@ -35,6 +35,7 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorStatus;
 import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.rpc.ServiceRetryPolicy;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -212,6 +213,31 @@ public interface OverlordClient
    */
   ListenableFuture<IndexingTotalWorkerCapacityInfo> getTotalWorkerCapacity();
 
+  /**
+   * Checks if compaction supervisors are enabled on the Overlord.
+   * When this returns true, the Coordinator does not run CompactSegments duty.
+   * <p>
+   * API: {@code /druid/indexer/v1/compaction/isSupervisorEnabled}
+   */
+  ListenableFuture<Boolean> isCompactionSupervisorEnabled();
+
+  /**
+   * Gets the number of bytes yet to be compacted for the given datasource.
+   * <p>
+   * API: {@code /druid/indexer/v1/compaction/progress}
+   */
+  ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource);
+
+  /**
+   * Gets the latest compaction snapshots of one or all datasources.
+   * <p>
+   * API: {@code /druid/indexer/v1/compaction/status}
+   *
+   * @param dataSource If passed as non-null, then the returned list contains only
+   *                   the snapshot for this datasource.
+   */
+  ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource);
+
   /**
    * Returns a copy of this client with a different retry policy.
    */
diff --git a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java
index 3e3d86ca5f2..2a48e0ed692 100644
--- a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java
+++ b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java
@@ -45,6 +45,7 @@ import org.apache.druid.rpc.IgnoreHttpResponseHandler;
 import org.apache.druid.rpc.RequestBuilder;
 import org.apache.druid.rpc.ServiceClient;
 import org.apache.druid.rpc.ServiceRetryPolicy;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.joda.time.Interval;
 
@@ -312,6 +313,52 @@ public class OverlordClientImpl implements OverlordClient
     );
   }
 
+  @Override
+  public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+  {
+    final StringBuilder pathBuilder = new StringBuilder("/druid/indexer/v1/compaction/status");
+    if (dataSource != null && !dataSource.isEmpty()) {
+      pathBuilder.append("?").append("dataSource=").append(dataSource);
+    }
+
+    return FutureUtils.transform(
+        client.asyncRequest(
+            new RequestBuilder(HttpMethod.GET, pathBuilder.toString()),
+            new BytesFullResponseHandler()
+        ),
+        holder -> JacksonUtils.readValue(
+            jsonMapper,
+            holder.getContent(),
+            new TypeReference<List<AutoCompactionSnapshot>>() {}
+        )
+    );
+  }
+
+  @Override
+  public ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource)
+  {
+    final String path = "/druid/indexer/v1/compaction/progress?dataSource=" + dataSource;
+    return FutureUtils.transform(
+        client.asyncRequest(
+            new RequestBuilder(HttpMethod.GET, path),
+            new BytesFullResponseHandler()
+        ),
+        holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), Long.class)
+    );
+  }
+
+  @Override
+  public ListenableFuture<Boolean> isCompactionSupervisorEnabled()
+  {
+    return FutureUtils.transform(
+        client.asyncRequest(
+            new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/isSupervisorEnabled"),
+            new BytesFullResponseHandler()
+        ),
+        holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), Boolean.class)
+    );
+  }
+
   @Override
   public OverlordClientImpl withRetryPolicy(ServiceRetryPolicy retryPolicy)
   {
diff --git a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java
new file mode 100644
index 00000000000..21f6cafbfaf
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.druid.java.util.common.guava.Comparators;
+
+import javax.annotation.Nullable;
+import java.util.Comparator;
+import java.util.Objects;
+
+/**
+ * Base implementation of {@link CompactionCandidateSearchPolicy} that can have
+ * a {@code priorityDatasource}.
+ */
+public abstract class BaseCandidateSearchPolicy
+    implements CompactionCandidateSearchPolicy, Comparator<CompactionCandidate>
+{
+  private final String priorityDatasource;
+  private final Comparator<CompactionCandidate> comparator;
+
+  protected BaseCandidateSearchPolicy(@Nullable String priorityDatasource)
+  {
+    this.priorityDatasource = priorityDatasource;
+    if (priorityDatasource == null || priorityDatasource.isEmpty()) {
+      this.comparator = getSegmentComparator();
+    } else {
+      this.comparator = Comparators.alwaysFirst(priorityDatasource)
+                                   .onResultOf(CompactionCandidate::getDataSource)
+                                   .thenComparing(getSegmentComparator());
+    }
+  }
+
+  /**
+   * The candidates of this datasource are prioritized over all others.
+   */
+  @Nullable
+  @JsonProperty
+  public final String getPriorityDatasource()
+  {
+    return priorityDatasource;
+  }
+
+  @Override
+  public final int compare(CompactionCandidate o1, CompactionCandidate o2)
+  {
+    return comparator.compare(o1, o2);
+  }
+
+  @Override
+  public boolean isEligibleForCompaction(
+      CompactionCandidate candidate,
+      CompactionStatus currentCompactionStatus,
+      CompactionTaskStatus latestTaskStatus
+  )
+  {
+    return true;
+  }
+
+  /**
+   * Compares between two compaction candidates. Used to determine the
+   * order in which segments and intervals should be picked for compaction.
+   */
+  protected abstract Comparator<CompactionCandidate> getSegmentComparator();
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    BaseCandidateSearchPolicy that = (BaseCandidateSearchPolicy) o;
+    return Objects.equals(this.priorityDatasource, that.priorityDatasource);
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hashCode(priorityDatasource);
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/compact/SegmentsToCompact.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java
similarity index 55%
rename from server/src/main/java/org/apache/druid/server/coordinator/compact/SegmentsToCompact.java
rename to server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java
index 27ce9beab81..4cd9b22df81 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/compact/SegmentsToCompact.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java
@@ -17,53 +17,42 @@
  * under the License.
  */
 
-package org.apache.druid.server.coordinator.compact;
+package org.apache.druid.server.compaction;
 
+import org.apache.druid.error.InvalidInput;
 import org.apache.druid.java.util.common.JodaUtils;
 import org.apache.druid.segment.SegmentUtils;
 import org.apache.druid.timeline.DataSegment;
 import org.joda.time.Interval;
 
-import java.util.Collections;
+import javax.annotation.Nullable;
 import java.util.List;
-import java.util.NoSuchElementException;
 import java.util.stream.Collectors;
 
 /**
- * List of segments to compact.
+ * Non-empty list of segments of a datasource being considered for compaction.
+ * A candidate typically contains all the segments of a single time chunk.
  */
-public class SegmentsToCompact
+public class CompactionCandidate
 {
-  private static final SegmentsToCompact EMPTY_INSTANCE = new SegmentsToCompact();
-
   private final List<DataSegment> segments;
   private final Interval umbrellaInterval;
+  private final String dataSource;
   private final long totalBytes;
   private final int numIntervals;
 
-  static SegmentsToCompact empty()
-  {
-    return EMPTY_INSTANCE;
-  }
+  private final CompactionStatus currentStatus;
 
-  public static SegmentsToCompact from(List<DataSegment> segments)
+  public static CompactionCandidate from(List<DataSegment> segments)
   {
     if (segments == null || segments.isEmpty()) {
-      return empty();
+      throw InvalidInput.exception("Segments to compact must be non-empty");
     } else {
-      return new SegmentsToCompact(segments);
+      return new CompactionCandidate(segments, null);
     }
   }
 
-  private SegmentsToCompact()
-  {
-    this.segments = Collections.emptyList();
-    this.totalBytes = 0L;
-    this.numIntervals = 0;
-    this.umbrellaInterval = null;
-  }
-
-  private SegmentsToCompact(List<DataSegment> segments)
+  private CompactionCandidate(List<DataSegment> segments, @Nullable CompactionStatus currentStatus)
   {
     this.segments = segments;
     this.totalBytes = segments.stream().mapToLong(DataSegment::getSize).sum();
@@ -71,53 +60,72 @@ public class SegmentsToCompact
         segments.stream().map(DataSegment::getInterval).collect(Collectors.toList())
     );
     this.numIntervals = (int) segments.stream().map(DataSegment::getInterval).distinct().count();
+    this.dataSource = segments.get(0).getDataSource();
+    this.currentStatus = currentStatus;
   }
 
+  /**
+   * @return Non-empty list of segments that make up this candidate.
+   */
   public List<DataSegment> getSegments()
   {
     return segments;
   }
 
-  public DataSegment getFirst()
-  {
-    if (segments.isEmpty()) {
-      throw new NoSuchElementException("No segment to compact");
-    } else {
-      return segments.get(0);
-    }
-  }
-
-  public boolean isEmpty()
-  {
-    return segments.isEmpty();
-  }
-
   public long getTotalBytes()
   {
     return totalBytes;
   }
 
-  public int size()
+  public int numSegments()
   {
     return segments.size();
   }
 
+  /**
+   * Umbrella interval of all the segments in this candidate. This typically
+   * corresponds to a single time chunk in the segment timeline.
+   */
   public Interval getUmbrellaInterval()
   {
     return umbrellaInterval;
   }
 
+  public String getDataSource()
+  {
+    return dataSource;
+  }
+
   public CompactionStatistics getStats()
   {
-    return CompactionStatistics.create(totalBytes, size(), numIntervals);
+    return CompactionStatistics.create(totalBytes, numSegments(), numIntervals);
+  }
+
+  /**
+   * Current compaction status of the time chunk corresponding to this candidate.
+   */
+  @Nullable
+  public CompactionStatus getCurrentStatus()
+  {
+    return currentStatus;
+  }
+
+  /**
+   * Creates a copy of this CompactionCandidate object with the given status.
+   */
+  public CompactionCandidate withCurrentStatus(CompactionStatus status)
+  {
+    return new CompactionCandidate(this.segments, status);
   }
 
   @Override
   public String toString()
   {
     return "SegmentsToCompact{" +
-           "segments=" + SegmentUtils.commaSeparatedIdentifiers(segments) +
+           "datasource=" + dataSource +
+           ", segments=" + SegmentUtils.commaSeparatedIdentifiers(segments) +
            ", totalSize=" + totalBytes +
+           ", currentStatus=" + currentStatus +
            '}';
   }
 }
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java
new file mode 100644
index 00000000000..cf0e016f054
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.druid.server.coordinator.duty.CompactSegments;
+
+import java.util.Comparator;
+
+/**
+ * Policy used by {@link CompactSegments} duty to pick segments for compaction.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonSubTypes(value = {
+    @JsonSubTypes.Type(name = "newestSegmentFirst", value = NewestSegmentFirstPolicy.class)
+})
+public interface CompactionCandidateSearchPolicy extends Comparator<CompactionCandidate>
+{
+  /**
+   * Compares between two compaction candidates. Used to determine the
+   * order in which segments and intervals should be picked for compaction.
+   */
+  @Override
+  int compare(CompactionCandidate o1, CompactionCandidate o2);
+
+  /**
+   * Checks if the given {@link CompactionCandidate} is eligible for compaction
+   * in the current iteration. A policy may implement this method to skip
+   * compacting intervals or segments that do not fulfil some required criteria.
+   */
+  boolean isEligibleForCompaction(
+      CompactionCandidate candidate,
+      CompactionStatus currentCompactionStatus,
+      CompactionTaskStatus latestTaskStatus
+  );
+}
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
new file mode 100644
index 00000000000..6e8ec6b4b11
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.druid.client.indexing.ClientCompactionTaskQuery;
+import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig;
+import org.apache.druid.client.indexing.IndexingTotalWorkerCapacityInfo;
+import org.apache.druid.client.indexing.IndexingWorkerInfo;
+import org.apache.druid.client.indexing.TaskPayloadResponse;
+import org.apache.druid.client.indexing.TaskStatusResponse;
+import org.apache.druid.indexer.TaskStatus;
+import org.apache.druid.indexer.TaskStatusPlus;
+import org.apache.druid.indexer.report.TaskReport;
+import org.apache.druid.indexing.overlord.supervisor.SupervisorStatus;
+import org.apache.druid.java.util.common.parsers.CloseableIterator;
+import org.apache.druid.metadata.LockFilterPolicy;
+import org.apache.druid.rpc.ServiceRetryPolicy;
+import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.coordinator.ClusterCompactionConfig;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+import org.apache.druid.server.coordinator.DruidCompactionConfig;
+import org.apache.druid.server.coordinator.duty.CompactSegments;
+import org.apache.druid.server.coordinator.stats.CoordinatorRunStats;
+import org.apache.druid.timeline.SegmentTimeline;
+import org.joda.time.Interval;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Simulates runs of auto-compaction duty to obtain the expected list of
+ * compaction tasks that would be submitted by the actual compaction duty.
+ */
+public class CompactionRunSimulator
+{
+  private final CompactionStatusTracker statusTracker;
+  private final OverlordClient readOnlyOverlordClient;
+
+  public CompactionRunSimulator(
+      CompactionStatusTracker statusTracker,
+      OverlordClient overlordClient
+  )
+  {
+    this.statusTracker = statusTracker;
+    this.readOnlyOverlordClient = new ReadOnlyOverlordClient(overlordClient);
+  }
+
+  /**
+   * Simulates a run of the compact segments duty with the given compaction config
+   * assuming unlimited compaction task slots.
+   */
+  public CompactionSimulateResult simulateRunWithConfig(
+      DruidCompactionConfig compactionConfig,
+      Map<String, SegmentTimeline> datasourceTimelines
+  )
+  {
+    final Table compactedIntervals
+        = Table.withColumnNames("dataSource", "interval", "numSegments", "bytes");
+    final Table runningIntervals
+        = Table.withColumnNames("dataSource", "interval", "numSegments", "bytes", "maxTaskSlots", "reasonToCompact");
+    final Table queuedIntervals
+        = Table.withColumnNames("dataSource", "interval", "numSegments", "bytes", "maxTaskSlots", "reasonToCompact");
+    final Table skippedIntervals
+        = Table.withColumnNames("dataSource", "interval", "numSegments", "bytes", "reasonToSkip");
+
+    // Add a read-only wrapper over the actual status tracker so that we can
+    // account for the active tasks
+    final CompactionStatusTracker simulationStatusTracker = new CompactionStatusTracker(null)
+    {
+      @Override
+      public CompactionStatus computeCompactionStatus(
+          CompactionCandidate candidate,
+          DataSourceCompactionConfig config,
+          CompactionCandidateSearchPolicy searchPolicy
+      )
+      {
+        return statusTracker.computeCompactionStatus(candidate, config, searchPolicy);
+      }
+
+      @Override
+      public void onCompactionStatusComputed(
+          CompactionCandidate candidateSegments,
+          DataSourceCompactionConfig config
+      )
+      {
+        final CompactionStatus status = candidateSegments.getCurrentStatus();
+        if (status.getState() == CompactionStatus.State.COMPLETE) {
+          compactedIntervals.addRow(
+              createRow(candidateSegments, null, null)
+          );
+        } else if (status.getState() == CompactionStatus.State.RUNNING) {
+          runningIntervals.addRow(
+              createRow(candidateSegments, ClientCompactionTaskQueryTuningConfig.from(config), status.getReason())
+          );
+        } else if (status.getState() == CompactionStatus.State.SKIPPED) {
+          skippedIntervals.addRow(
+              createRow(candidateSegments, null, status.getReason())
+          );
+        }
+      }
+
+      @Override
+      public void onTaskSubmitted(ClientCompactionTaskQuery taskPayload, CompactionCandidate candidateSegments)
+      {
+        // Add a row for each task in order of submission
+        final CompactionStatus status = candidateSegments.getCurrentStatus();
+        queuedIntervals.addRow(
+            createRow(candidateSegments, taskPayload.getTuningConfig(), status.getReason())
+        );
+      }
+    };
+
+    // Unlimited task slots to ensure that simulator does not skip any interval
+    final DruidCompactionConfig configWithUnlimitedTaskSlots = compactionConfig.withClusterConfig(
+        new ClusterCompactionConfig(1.0, Integer.MAX_VALUE, null, null, null)
+    );
+
+    final CoordinatorRunStats stats = new CoordinatorRunStats();
+    new CompactSegments(simulationStatusTracker, readOnlyOverlordClient).run(
+        configWithUnlimitedTaskSlots,
+        datasourceTimelines,
+        stats
+    );
+
+    final Map<CompactionStatus.State, Table> compactionStates = new HashMap<>();
+    if (!compactedIntervals.isEmpty()) {
+      compactionStates.put(CompactionStatus.State.COMPLETE, compactedIntervals);
+    }
+    if (!runningIntervals.isEmpty()) {
+      compactionStates.put(CompactionStatus.State.RUNNING, runningIntervals);
+    }
+    if (!queuedIntervals.isEmpty()) {
+      compactionStates.put(CompactionStatus.State.PENDING, queuedIntervals);
+    }
+    if (!skippedIntervals.isEmpty()) {
+      compactionStates.put(CompactionStatus.State.SKIPPED, skippedIntervals);
+    }
+
+    return new CompactionSimulateResult(compactionStates);
+  }
+
+  private Object[] createRow(
+      CompactionCandidate candidate,
+      ClientCompactionTaskQueryTuningConfig tuningConfig,
+      String reason
+  )
+  {
+    final List<Object> row = new ArrayList<>();
+    row.add(candidate.getDataSource());
+    row.add(candidate.getUmbrellaInterval());
+    row.add(candidate.numSegments());
+    row.add(candidate.getTotalBytes());
+    if (tuningConfig != null) {
+      row.add(CompactSegments.findMaxNumTaskSlotsUsedByOneNativeCompactionTask(tuningConfig));
+    }
+    if (reason != null) {
+      row.add(reason);
+    }
+
+    return row.toArray(new Object[0]);
+  }
+
+  /**
+   * Dummy overlord client that returns empty results for all APIs.
+   */
+  private static class ReadOnlyOverlordClient implements OverlordClient
+  {
+    final OverlordClient delegate;
+
+    ReadOnlyOverlordClient(OverlordClient delegate)
+    {
+      this.delegate = delegate;
+    }
+
+    @Override
+    public ListenableFuture<CloseableIterator<TaskStatusPlus>> taskStatuses(
+        @Nullable String state,
+        @Nullable String dataSource,
+        @Nullable Integer maxCompletedTasks
+    )
+    {
+      return delegate.taskStatuses(state, dataSource, maxCompletedTasks);
+    }
+
+    @Override
+    public ListenableFuture<Map<String, TaskStatus>> taskStatuses(Set<String> taskIds)
+    {
+      return delegate.taskStatuses(taskIds);
+    }
+
+    @Override
+    public ListenableFuture<TaskPayloadResponse> taskPayload(String taskId)
+    {
+      return delegate.taskPayload(taskId);
+    }
+
+    @Override
+    public ListenableFuture<Map<String, List<Interval>>> findLockedIntervals(List<LockFilterPolicy> lockFilterPolicies)
+    {
+      return delegate.findLockedIntervals(lockFilterPolicies);
+    }
+
+    @Override
+    public ListenableFuture<IndexingTotalWorkerCapacityInfo> getTotalWorkerCapacity()
+    {
+      // Unlimited worker capacity to ensure that simulator does not skip any interval
+      return Futures.immediateFuture(
+          new IndexingTotalWorkerCapacityInfo(Integer.MAX_VALUE, Integer.MAX_VALUE)
+      );
+    }
+
+    @Override
+    public ListenableFuture<Void> runTask(String taskId, Object taskObject)
+    {
+      return Futures.immediateVoidFuture();
+    }
+
+    @Override
+    public ListenableFuture<Void> cancelTask(String taskId)
+    {
+      return Futures.immediateVoidFuture();
+    }
+
+    // Unsupported methods as these are not used by the CompactionScheduler / CompactSegments duty
+
+    @Override
+    public ListenableFuture<URI> findCurrentLeader()
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListenableFuture<TaskStatusResponse> taskStatus(String taskId)
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListenableFuture<TaskReport.ReportMap> taskReportAsMap(String taskId)
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListenableFuture<CloseableIterator<SupervisorStatus>> supervisorStatuses()
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListenableFuture<Integer> killPendingSegments(String dataSource, Interval interval)
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListenableFuture<List<IndexingWorkerInfo>> getWorkers()
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource)
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListenableFuture<Boolean> isCompactionSupervisorEnabled()
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public OverlordClient withRetryPolicy(ServiceRetryPolicy retryPolicy)
+    {
+      return this;
+    }
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionSegmentSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionSegmentIterator.java
similarity index 56%
rename from server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionSegmentSearchPolicy.java
rename to server/src/main/java/org/apache/druid/server/compaction/CompactionSegmentIterator.java
index cc5f4f59d85..295aa2881ad 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionSegmentSearchPolicy.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionSegmentIterator.java
@@ -17,27 +17,27 @@
  * under the License.
  */
 
-package org.apache.druid.server.coordinator.compact;
-
-import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
-import org.apache.druid.server.coordinator.duty.CompactSegments;
-import org.apache.druid.timeline.SegmentTimeline;
-import org.joda.time.Interval;
+package org.apache.druid.server.compaction;
 
+import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 
 /**
- * Segment searching policy used by {@link CompactSegments}.
+ * Iterator over compactible segments.
  */
-public interface CompactionSegmentSearchPolicy
+public interface CompactionSegmentIterator extends Iterator<CompactionCandidate>
 {
   /**
-   * Creates an iterator that returns compactible segments.
+   * List of candidate segments that are already compacted and do not need to be
+   * compacted again. None of these segments are returned by {@link #next()}.
    */
-  CompactionSegmentIterator createIterator(
-      Map<String, DataSourceCompactionConfig> compactionConfigs,
-      Map<String, SegmentTimeline> dataSources,
-      Map<String, List<Interval>> skipIntervals
-  );
+  List<CompactionCandidate> getCompactedSegments();
+
+  /**
+   * List of candidate segments that have been skipped for compaction as they
+   * cannot be compacted due to some reason. None of these segments are returned
+   * by {@link #next()}.
+   */
+  List<CompactionCandidate> getSkippedSegments();
+
 }
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionSimulateResult.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionSimulateResult.java
new file mode 100644
index 00000000000..7a48ccf0e5b
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionSimulateResult.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Map;
+
+public class CompactionSimulateResult
+{
+  private final Map<CompactionStatus.State, Table> compactionStates;
+
+  @JsonCreator
+  public CompactionSimulateResult(
+      @JsonProperty("compactionStates") Map<CompactionStatus.State, Table> compactionStates
+  )
+  {
+    this.compactionStates = compactionStates;
+  }
+
+  @JsonProperty
+  public Map<CompactionStatus.State, Table> getCompactionStates()
+  {
+    return compactionStates;
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionStatistics.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatistics.java
similarity index 96%
rename from server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionStatistics.java
rename to server/src/main/java/org/apache/druid/server/compaction/CompactionStatistics.java
index 6997dec47c0..23f1b7fe9ef 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionStatistics.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatistics.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.druid.server.coordinator.compact;
+package org.apache.druid.server.compaction;
 
 /**
  * Used to track statistics for segments in different states of compaction.
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java
similarity index 70%
rename from server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionStatus.java
rename to server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java
index fa053fb8d6a..9e9a199917a 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionStatus.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java
@@ -17,20 +17,22 @@
  * under the License.
  */
 
-package org.apache.druid.server.coordinator.compact;
+package org.apache.druid.server.compaction;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.druid.client.indexing.ClientCompactionTaskGranularitySpec;
 import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig;
 import org.apache.druid.client.indexing.ClientCompactionTaskTransformSpec;
 import org.apache.druid.common.config.Configs;
 import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
 import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.indexer.partitions.HashedPartitionsSpec;
 import org.apache.druid.indexer.partitions.PartitionsSpec;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.java.util.common.granularity.GranularityType;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
@@ -44,11 +46,16 @@ import java.util.List;
 import java.util.function.Function;
 
 /**
- * Represents the status of compaction for a given list of candidate segments.
+ * Represents the status of compaction for a given {@link CompactionCandidate}.
  */
 public class CompactionStatus
 {
-  private static final CompactionStatus COMPLETE = new CompactionStatus(true, null);
+  private static final CompactionStatus COMPLETE = new CompactionStatus(State.COMPLETE, null);
+
+  public enum State
+  {
+    COMPLETE, PENDING, RUNNING, SKIPPED
+  }
 
   /**
    * List of checks performed to determine if compaction is already complete.
@@ -68,54 +75,133 @@ public class CompactionStatus
       Evaluator::transformSpecFilterIsUpToDate
   );
 
-  private final boolean complete;
-  private final String reasonToCompact;
+  private final State state;
+  private final String reason;
 
-  private CompactionStatus(boolean complete, String reason)
+  private CompactionStatus(State state, String reason)
   {
-    this.complete = complete;
-    this.reasonToCompact = reason;
+    this.state = state;
+    this.reason = reason;
   }
 
   public boolean isComplete()
   {
-    return complete;
+    return state == State.COMPLETE;
   }
 
-  public String getReasonToCompact()
+  public boolean isSkipped()
   {
-    return reasonToCompact;
+    return state == State.SKIPPED;
+  }
+
+  public String getReason()
+  {
+    return reason;
+  }
+
+  public State getState()
+  {
+    return state;
+  }
+
+  @Override
+  public String toString()
+  {
+    return "CompactionStatus{" +
+           "state=" + state +
+           ", reason=" + reason +
+           '}';
   }
 
   private static CompactionStatus incomplete(String reasonFormat, Object... args)
   {
-    return new CompactionStatus(false, StringUtils.format(reasonFormat, args));
+    return new CompactionStatus(State.PENDING, StringUtils.format(reasonFormat, args));
   }
 
-  private static CompactionStatus completeIfEqual(String field, Object configured, Object current)
+  private static <T> CompactionStatus completeIfEqual(
+      String field,
+      T configured,
+      T current,
+      Function<T, String> stringFunction
+  )
   {
     if (configured == null || configured.equals(current)) {
       return COMPLETE;
     } else {
-      return configChanged(field, configured, current);
+      return configChanged(field, configured, current, stringFunction);
     }
   }
 
-  private static CompactionStatus configChanged(String field, Object configured, Object current)
+  private static <T> CompactionStatus configChanged(
+      String field,
+      T target,
+      T current,
+      Function<T, String> stringFunction
+  )
   {
     return CompactionStatus.incomplete(
-        "Configured %s[%s] is different from current %s[%s]",
-        field, configured, field, current
+        "'%s' mismatch: required[%s], current[%s]",
+        field,
+        target == null ? null : stringFunction.apply(target),
+        current == null ? null : stringFunction.apply(current)
     );
   }
 
+  private static String asString(Granularity granularity)
+  {
+    if (granularity == null) {
+      return null;
+    }
+    for (GranularityType type : GranularityType.values()) {
+      if (type.getDefaultGranularity().equals(granularity)) {
+        return type.toString();
+      }
+    }
+    return granularity.toString();
+  }
+
+  private static String asString(PartitionsSpec partitionsSpec)
+  {
+    if (partitionsSpec instanceof DimensionRangePartitionsSpec) {
+      DimensionRangePartitionsSpec rangeSpec = (DimensionRangePartitionsSpec) partitionsSpec;
+      return StringUtils.format(
+          "'range' on %s with %,d rows",
+          rangeSpec.getPartitionDimensions(), rangeSpec.getTargetRowsPerSegment()
+      );
+    } else if (partitionsSpec instanceof HashedPartitionsSpec) {
+      HashedPartitionsSpec hashedSpec = (HashedPartitionsSpec) partitionsSpec;
+      return StringUtils.format(
+          "'hashed' on %s with %,d rows",
+          hashedSpec.getPartitionDimensions(), hashedSpec.getTargetRowsPerSegment()
+      );
+    } else if (partitionsSpec instanceof DynamicPartitionsSpec) {
+      DynamicPartitionsSpec dynamicSpec = (DynamicPartitionsSpec) partitionsSpec;
+      return StringUtils.format(
+          "'dynamic' with %,d rows",
+          dynamicSpec.getMaxRowsPerSegment()
+      );
+    } else {
+      return partitionsSpec.toString();
+    }
+  }
+
+  static CompactionStatus skipped(String reasonFormat, Object... args)
+  {
+    return new CompactionStatus(State.SKIPPED, StringUtils.format(reasonFormat, args));
+  }
+
+  static CompactionStatus running(String reasonForCompaction)
+  {
+    return new CompactionStatus(State.RUNNING, reasonForCompaction);
+  }
+
   /**
    * Determines the CompactionStatus of the given candidate segments by evaluating
    * the {@link #CHECKS} one by one. If any check returns an incomplete status,
    * further checks are not performed and the incomplete status is returned.
    */
-  static CompactionStatus of(
-      SegmentsToCompact candidateSegments,
+  static CompactionStatus compute(
+      CompactionCandidate candidateSegments,
       DataSourceCompactionConfig config,
       ObjectMapper objectMapper
   )
@@ -149,23 +235,21 @@ public class CompactionStatus
   {
     private final ObjectMapper objectMapper;
     private final DataSourceCompactionConfig compactionConfig;
-    private final SegmentsToCompact candidateSegments;
+    private final CompactionCandidate candidateSegments;
     private final CompactionState lastCompactionState;
     private final ClientCompactionTaskQueryTuningConfig tuningConfig;
     private final ClientCompactionTaskGranularitySpec existingGranularitySpec;
     private final UserCompactionTaskGranularityConfig configuredGranularitySpec;
 
     private Evaluator(
-        SegmentsToCompact candidateSegments,
+        CompactionCandidate candidateSegments,
         DataSourceCompactionConfig compactionConfig,
         ObjectMapper objectMapper
     )
     {
-      Preconditions.checkArgument(!candidateSegments.isEmpty(), "Empty candidates");
-
       this.candidateSegments = candidateSegments;
       this.objectMapper = objectMapper;
-      this.lastCompactionState = candidateSegments.getFirst().getLastCompactionState();
+      this.lastCompactionState = candidateSegments.getSegments().get(0).getLastCompactionState();
       this.compactionConfig = compactionConfig;
       this.tuningConfig = ClientCompactionTaskQueryTuningConfig.from(compactionConfig);
       this.configuredGranularitySpec = compactionConfig.getGranularitySpec();
@@ -182,7 +266,7 @@ public class CompactionStatus
     private CompactionStatus segmentsHaveBeenCompactedAtLeastOnce()
     {
       if (lastCompactionState == null) {
-        return CompactionStatus.incomplete("Not compacted yet");
+        return CompactionStatus.incomplete("not compacted yet");
       } else {
         return COMPLETE;
       }
@@ -196,7 +280,7 @@ public class CompactionStatus
       if (allHaveSameCompactionState) {
         return COMPLETE;
       } else {
-        return CompactionStatus.incomplete("Candidate segments have different last compaction states.");
+        return CompactionStatus.incomplete("segments have different last compaction states");
       }
     }
 
@@ -205,7 +289,8 @@ public class CompactionStatus
       return CompactionStatus.completeIfEqual(
           "partitionsSpec",
           findPartitionsSpecFromConfig(tuningConfig),
-          lastCompactionState.getPartitionsSpec()
+          lastCompactionState.getPartitionsSpec(),
+          CompactionStatus::asString
       );
     }
 
@@ -214,7 +299,8 @@ public class CompactionStatus
       return CompactionStatus.completeIfEqual(
           "indexSpec",
           Configs.valueOrDefault(tuningConfig.getIndexSpec(), IndexSpec.DEFAULT),
-          objectMapper.convertValue(lastCompactionState.getIndexSpec(), IndexSpec.class)
+          objectMapper.convertValue(lastCompactionState.getIndexSpec(), IndexSpec.class),
+          String::valueOf
       );
     }
 
@@ -239,15 +325,16 @@ public class CompactionStatus
         );
         if (needsCompaction) {
           return CompactionStatus.incomplete(
-              "Configured segmentGranularity[%s] does not align with segment intervals.",
-              configuredSegmentGranularity
+              "segmentGranularity: segments do not align with target[%s]",
+              asString(configuredSegmentGranularity)
           );
         }
       } else {
         return CompactionStatus.configChanged(
             "segmentGranularity",
             configuredSegmentGranularity,
-            existingSegmentGranularity
+            existingSegmentGranularity,
+            CompactionStatus::asString
         );
       }
 
@@ -262,7 +349,8 @@ public class CompactionStatus
         return CompactionStatus.completeIfEqual(
             "rollup",
             configuredGranularitySpec.isRollup(),
-            existingGranularitySpec == null ? null : existingGranularitySpec.isRollup()
+            existingGranularitySpec == null ? null : existingGranularitySpec.isRollup(),
+            String::valueOf
         );
       }
     }
@@ -275,7 +363,8 @@ public class CompactionStatus
         return CompactionStatus.completeIfEqual(
             "queryGranularity",
             configuredGranularitySpec.getQueryGranularity(),
-            existingGranularitySpec == null ? null : existingGranularitySpec.getQueryGranularity()
+            existingGranularitySpec == null ? null : existingGranularitySpec.getQueryGranularity(),
+            CompactionStatus::asString
         );
       }
     }
@@ -289,7 +378,8 @@ public class CompactionStatus
         return CompactionStatus.completeIfEqual(
             "dimensionsSpec",
             compactionConfig.getDimensionsSpec().getDimensions(),
-            existingDimensionsSpec == null ? null : existingDimensionsSpec.getDimensions()
+            existingDimensionsSpec == null ? null : existingDimensionsSpec.getDimensions(),
+            String::valueOf
         );
       }
     }
@@ -309,8 +399,9 @@ public class CompactionStatus
       if (existingMetricsSpec == null || !Arrays.deepEquals(configuredMetricsSpec, existingMetricsSpec)) {
         return CompactionStatus.configChanged(
             "metricsSpec",
-            Arrays.toString(configuredMetricsSpec),
-            Arrays.toString(existingMetricsSpec)
+            configuredMetricsSpec,
+            existingMetricsSpec,
+            Arrays::toString
         );
       } else {
         return COMPLETE;
@@ -330,7 +421,8 @@ public class CompactionStatus
       return CompactionStatus.completeIfEqual(
           "transformSpec filter",
           compactionConfig.getTransformSpec().getFilter(),
-          existingTransformSpec == null ? null : existingTransformSpec.getFilter()
+          existingTransformSpec == null ? null : existingTransformSpec.getFilter(),
+          String::valueOf
       );
     }
 
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java
new file mode 100644
index 00000000000..ab7ddbbb91a
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.inject.Inject;
+import org.apache.druid.client.indexing.ClientCompactionTaskQuery;
+import org.apache.druid.indexer.TaskState;
+import org.apache.druid.indexer.TaskStatus;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+import org.apache.druid.server.coordinator.DruidCompactionConfig;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.Interval;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Tracks status of recently submitted compaction tasks. Can be used by a segment
+ * search policy to skip an interval if it has been recently compacted or if it
+ * keeps failing repeatedly.
+ */
+public class CompactionStatusTracker
+{
+  private static final Duration MAX_STATUS_RETAIN_DURATION = Duration.standardHours(12);
+
+  private final ObjectMapper objectMapper;
+  private final ConcurrentHashMap<String, DatasourceStatus> datasourceStatuses
+      = new ConcurrentHashMap<>();
+  private final ConcurrentHashMap<String, CompactionCandidate> submittedTaskIdToSegments
+      = new ConcurrentHashMap<>();
+
+  @Inject
+  public CompactionStatusTracker(ObjectMapper objectMapper)
+  {
+    this.objectMapper = objectMapper;
+  }
+
+  public void stop()
+  {
+    datasourceStatuses.clear();
+  }
+
+  public void removeDatasource(String datasource)
+  {
+    datasourceStatuses.remove(datasource);
+  }
+
+  public CompactionTaskStatus getLatestTaskStatus(CompactionCandidate candidates)
+  {
+    return datasourceStatuses
+        .getOrDefault(candidates.getDataSource(), DatasourceStatus.EMPTY)
+        .intervalToTaskStatus
+        .get(candidates.getUmbrellaInterval());
+  }
+
+  /**
+   * Set of submitted compaction task IDs which have not been marked completed
+   * via {@link #onTaskFinished} yet.
+   */
+  public Set<String> getSubmittedTaskIds()
+  {
+    return submittedTaskIdToSegments.keySet();
+  }
+
+  public CompactionStatus computeCompactionStatus(
+      CompactionCandidate candidate,
+      DataSourceCompactionConfig config,
+      CompactionCandidateSearchPolicy searchPolicy
+  )
+  {
+    final CompactionStatus compactionStatus = CompactionStatus.compute(candidate, config, objectMapper);
+    if (compactionStatus.isComplete()) {
+      return compactionStatus;
+    }
+
+    // Skip intervals that violate max allowed input segment size
+    final long inputSegmentSize = config.getInputSegmentSizeBytes();
+    if (candidate.getTotalBytes() > inputSegmentSize) {
+      return CompactionStatus.skipped(
+          "'inputSegmentSize' exceeded: Total segment size[%d] is larger than allowed inputSegmentSize[%d]",
+          candidate.getTotalBytes(), inputSegmentSize
+      );
+    }
+
+    // Skip intervals that already have a running task
+    final CompactionTaskStatus lastTaskStatus = getLatestTaskStatus(candidate);
+    if (lastTaskStatus != null && lastTaskStatus.getState() == TaskState.RUNNING) {
+      return CompactionStatus.skipped("Task for interval is already running");
+    }
+
+    // Skip intervals that have been filtered out by the policy
+    if (!searchPolicy.isEligibleForCompaction(candidate, compactionStatus, lastTaskStatus)) {
+      return CompactionStatus.skipped("Rejected by search policy");
+    }
+
+    return compactionStatus;
+  }
+
+  public void onCompactionStatusComputed(
+      CompactionCandidate candidateSegments,
+      DataSourceCompactionConfig config
+  )
+  {
+    // Nothing to do, used by simulator
+  }
+
+  public void onCompactionConfigUpdated(DruidCompactionConfig compactionConfig)
+  {
+    final Set<String> compactionEnabledDatasources = new HashSet<>();
+    if (compactionConfig.getCompactionConfigs() != null) {
+      compactionConfig.getCompactionConfigs().forEach(config -> {
+        getOrComputeDatasourceStatus(config.getDataSource())
+            .cleanupStaleTaskStatuses();
+
+        compactionEnabledDatasources.add(config.getDataSource());
+      });
+    }
+
+    // Clean up state for datasources where compaction has been disabled
+    final Set<String> allDatasources = new HashSet<>(datasourceStatuses.keySet());
+    allDatasources.forEach(datasource -> {
+      if (!compactionEnabledDatasources.contains(datasource)) {
+        datasourceStatuses.remove(datasource);
+      }
+    });
+  }
+
+  public void onTaskSubmitted(
+      ClientCompactionTaskQuery taskPayload,
+      CompactionCandidate candidateSegments
+  )
+  {
+    submittedTaskIdToSegments.put(taskPayload.getId(), candidateSegments);
+    getOrComputeDatasourceStatus(taskPayload.getDataSource())
+        .handleSubmittedTask(candidateSegments);
+  }
+
+  public void onTaskFinished(String taskId, TaskStatus taskStatus)
+  {
+    if (!taskStatus.isComplete()) {
+      return;
+    }
+
+    final CompactionCandidate candidateSegments = submittedTaskIdToSegments.remove(taskId);
+    if (candidateSegments == null) {
+      // Nothing to do since we don't know the corresponding datasource or interval
+      return;
+    }
+
+    final Interval compactionInterval = candidateSegments.getUmbrellaInterval();
+    getOrComputeDatasourceStatus(candidateSegments.getDataSource())
+        .handleCompletedTask(compactionInterval, taskStatus);
+  }
+
+  private DatasourceStatus getOrComputeDatasourceStatus(String datasource)
+  {
+    return datasourceStatuses.computeIfAbsent(datasource, ds -> new DatasourceStatus());
+  }
+
+  /**
+   * Contains compaction task status of intervals of a datasource.
+   */
+  private static class DatasourceStatus
+  {
+    static final DatasourceStatus EMPTY = new DatasourceStatus();
+
+    final ConcurrentHashMap<Interval, CompactionTaskStatus> intervalToTaskStatus
+        = new ConcurrentHashMap<>();
+
+    void handleCompletedTask(Interval compactionInterval, TaskStatus taskStatus)
+    {
+      final CompactionTaskStatus lastKnownStatus = intervalToTaskStatus.get(compactionInterval);
+      final DateTime now = DateTimes.nowUtc();
+
+      final CompactionTaskStatus updatedStatus;
+      if (taskStatus.isSuccess()) {
+        updatedStatus = new CompactionTaskStatus(TaskState.SUCCESS, now, 0);
+      } else if (lastKnownStatus == null || lastKnownStatus.getState().isSuccess()) {
+        // This is the first failure
+        updatedStatus = new CompactionTaskStatus(TaskState.FAILED, now, 1);
+      } else {
+        updatedStatus = new CompactionTaskStatus(
+            TaskState.FAILED,
+            now,
+            lastKnownStatus.getNumConsecutiveFailures() + 1
+        );
+      }
+      intervalToTaskStatus.put(compactionInterval, updatedStatus);
+    }
+
+    void handleSubmittedTask(CompactionCandidate candidateSegments)
+    {
+      final Interval interval = candidateSegments.getUmbrellaInterval();
+      final CompactionTaskStatus lastStatus = intervalToTaskStatus.get(interval);
+
+      final DateTime now = DateTimes.nowUtc();
+      if (lastStatus == null || !lastStatus.getState().isFailure()) {
+        intervalToTaskStatus.put(interval, new CompactionTaskStatus(TaskState.RUNNING, now, 0));
+      } else {
+        intervalToTaskStatus.put(
+            interval,
+            new CompactionTaskStatus(TaskState.RUNNING, now, lastStatus.getNumConsecutiveFailures())
+        );
+      }
+    }
+
+    void cleanupStaleTaskStatuses()
+    {
+      final DateTime now = DateTimes.nowUtc();
+
+      final Set<Interval> staleIntervals = new HashSet<>();
+      intervalToTaskStatus.forEach((interval, taskStatus) -> {
+        if (taskStatus.getUpdatedTime().plus(MAX_STATUS_RETAIN_DURATION).isBefore(now)) {
+          staleIntervals.add(interval);
+        }
+      });
+
+      staleIntervals.forEach(intervalToTaskStatus::remove);
+    }
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionTaskStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionTaskStatus.java
new file mode 100644
index 00000000000..3431de706f6
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionTaskStatus.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import org.apache.druid.indexer.TaskState;
+import org.joda.time.DateTime;
+
+public class CompactionTaskStatus
+{
+  private final TaskState state;
+  private final DateTime updatedTime;
+  private final int numConsecutiveFailures;
+
+  public CompactionTaskStatus(
+      TaskState state,
+      DateTime updatedTime,
+      int numConsecutiveFailures
+  )
+  {
+    this.state = state;
+    this.updatedTime = updatedTime;
+    this.numConsecutiveFailures = numConsecutiveFailures;
+  }
+
+  public TaskState getState()
+  {
+    return state;
+  }
+
+  public DateTime getUpdatedTime()
+  {
+    return updatedTime;
+  }
+
+  public int getNumConsecutiveFailures()
+  {
+    return numConsecutiveFailures;
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/compact/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java
similarity index 73%
rename from server/src/main/java/org/apache/druid/server/coordinator/compact/DataSourceCompactibleSegmentIterator.java
rename to server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java
index c086be3112b..0abbd00e975 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/compact/DataSourceCompactibleSegmentIterator.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java
@@ -17,9 +17,8 @@
  * under the License.
  */
 
-package org.apache.druid.server.coordinator.compact;
+package org.apache.druid.server.compaction;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
@@ -37,6 +36,7 @@ import org.apache.druid.timeline.TimelineObjectHolder;
 import org.apache.druid.timeline.partition.NumberedPartitionChunk;
 import org.apache.druid.timeline.partition.NumberedShardSpec;
 import org.apache.druid.timeline.partition.PartitionChunk;
+import org.apache.druid.utils.CollectionUtils;
 import org.apache.druid.utils.Streams;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
@@ -59,67 +59,86 @@ import java.util.stream.Collectors;
 /**
  * Iterator over compactible segments of a datasource in order of specified priority.
  */
-public class DataSourceCompactibleSegmentIterator implements Iterator<SegmentsToCompact>
+public class DataSourceCompactibleSegmentIterator implements CompactionSegmentIterator
 {
   private static final Logger log = new Logger(DataSourceCompactibleSegmentIterator.class);
 
   private final String dataSource;
-  private final ObjectMapper objectMapper;
   private final DataSourceCompactionConfig config;
-  private final CompactionStatistics compactedSegmentStats = new CompactionStatistics();
-  private final CompactionStatistics skippedSegmentStats = new CompactionStatistics();
+  private final CompactionStatusTracker statusTracker;
+  private final CompactionCandidateSearchPolicy searchPolicy;
+
+  private final List<CompactionCandidate> compactedSegments = new ArrayList<>();
+  private final List<CompactionCandidate> skippedSegments = new ArrayList<>();
 
   // This is needed for datasource that has segmentGranularity configured
   // If configured segmentGranularity in config is finer than current segmentGranularity, the same set of segments
   // can belong to multiple intervals in the timeline. We keep track of the compacted intervals between each
   // run of the compaction job and skip any interval that was already previously compacted.
-  private final Set<Interval> compactedIntervals = new HashSet<>();
+  private final Set<Interval> queuedIntervals = new HashSet<>();
 
-  private final PriorityQueue<SegmentsToCompact> queue;
+  private final PriorityQueue<CompactionCandidate> queue;
 
   public DataSourceCompactibleSegmentIterator(
       DataSourceCompactionConfig config,
       SegmentTimeline timeline,
       List<Interval> skipIntervals,
-      Comparator<SegmentsToCompact> segmentPriority,
-      ObjectMapper objectMapper
+      CompactionCandidateSearchPolicy searchPolicy,
+      CompactionStatusTracker statusTracker
   )
   {
-    this.objectMapper = objectMapper;
+    this.statusTracker = statusTracker;
     this.config = config;
     this.dataSource = config.getDataSource();
-    this.queue = new PriorityQueue<>(segmentPriority);
+    this.searchPolicy = searchPolicy;
+    this.queue = new PriorityQueue<>(searchPolicy);
+
     populateQueue(timeline, skipIntervals);
   }
 
   private void populateQueue(SegmentTimeline timeline, List<Interval> skipIntervals)
   {
     if (timeline != null) {
-      Granularity configuredSegmentGranularity = null;
       if (!timeline.isEmpty()) {
         SegmentTimeline originalTimeline = null;
-        if (config.getGranularitySpec() != null && config.getGranularitySpec().getSegmentGranularity() != null) {
-          String temporaryVersion = DateTimes.nowUtc().toString();
-          Map<Interval, Set<DataSegment>> intervalToPartitionMap = new HashMap<>();
-          configuredSegmentGranularity = config.getGranularitySpec().getSegmentGranularity();
-          // Create a new timeline to hold segments in the new configured segment granularity
-          SegmentTimeline timelineWithConfiguredSegmentGranularity = new SegmentTimeline();
-          Set<DataSegment> segments = timeline.findNonOvershadowedObjectsInInterval(Intervals.ETERNITY, Partitions.ONLY_COMPLETE);
+        if (config.getSegmentGranularity() != null) {
+          final Set<DataSegment> segments = timeline.findNonOvershadowedObjectsInInterval(
+              Intervals.ETERNITY,
+              Partitions.ONLY_COMPLETE
+          );
+
+          // Skip compaction if any segment has partial-eternity interval
+          // See https://github.com/apache/druid/issues/13208
+          final List<DataSegment> partialEternitySegments = new ArrayList<>();
           for (DataSegment segment : segments) {
-            // Convert original segmentGranularity to new granularities bucket by configuredSegmentGranularity
-            // For example, if the original is interval of 2020-01-28/2020-02-03 with WEEK granularity
-            // and the configuredSegmentGranularity is MONTH, the segment will be split to two segments
-            // of 2020-01/2020-02 and 2020-02/2020-03.
             if (Intervals.ETERNITY.getStart().equals(segment.getInterval().getStart())
                 || Intervals.ETERNITY.getEnd().equals(segment.getInterval().getEnd())) {
-              // This is to prevent the coordinator from crashing as raised in https://github.com/apache/druid/issues/13208
-              log.warn("Cannot compact datasource[%s] containing segments with partial-ETERNITY intervals", dataSource);
-              return;
-            }
-            for (Interval interval : configuredSegmentGranularity.getIterable(segment.getInterval())) {
-              intervalToPartitionMap.computeIfAbsent(interval, k -> new HashSet<>()).add(segment);
+              partialEternitySegments.add(segment);
             }
           }
+          if (!partialEternitySegments.isEmpty()) {
+            CompactionCandidate candidatesWithStatus = CompactionCandidate.from(partialEternitySegments).withCurrentStatus(
+                CompactionStatus.skipped("Segments have partial-eternity intervals")
+            );
+            skippedSegments.add(candidatesWithStatus);
+            statusTracker.onCompactionStatusComputed(candidatesWithStatus, config);
+            return;
+          }
+
+          // Convert original segmentGranularity to new granularities bucket by configuredSegmentGranularity
+          // For example, if the original is interval of 2020-01-28/2020-02-03 with WEEK granularity
+          // and the configuredSegmentGranularity is MONTH, the segment will be split to two segments
+          // of 2020-01/2020-02 and 2020-02/2020-03.
+          final SegmentTimeline timelineWithConfiguredSegmentGranularity = new SegmentTimeline();
+          final Map<Interval, Set<DataSegment>> intervalToPartitionMap = new HashMap<>();
+          for (DataSegment segment : segments) {
+            for (Interval interval : config.getSegmentGranularity().getIterable(segment.getInterval())) {
+              intervalToPartitionMap.computeIfAbsent(interval, k -> new HashSet<>())
+                                    .add(segment);
+            }
+          }
+
+          final String temporaryVersion = DateTimes.nowUtc().toString();
           for (Map.Entry<Interval, Set<DataSegment>> partitionsPerInterval : intervalToPartitionMap.entrySet()) {
             Interval interval = partitionsPerInterval.getKey();
             int partitionNum = 0;
@@ -149,13 +168,7 @@ public class DataSourceCompactibleSegmentIterator implements Iterator<SegmentsTo
           originalTimeline = timeline;
           timeline = timelineWithConfiguredSegmentGranularity;
         }
-        final List<Interval> searchIntervals = findInitialSearchInterval(
-            dataSource,
-            timeline,
-            config.getSkipOffsetFromLatest(),
-            configuredSegmentGranularity,
-            skipIntervals
-        );
+        final List<Interval> searchIntervals = findInitialSearchInterval(timeline, skipIntervals);
         if (!searchIntervals.isEmpty()) {
           findAndEnqueueSegmentsToCompact(
               new CompactibleSegmentIterator(timeline, searchIntervals, originalTimeline)
@@ -167,14 +180,16 @@ public class DataSourceCompactibleSegmentIterator implements Iterator<SegmentsTo
     }
   }
 
-  public CompactionStatistics totalCompactedStatistics()
+  @Override
+  public List<CompactionCandidate> getCompactedSegments()
   {
-    return compactedSegmentStats;
+    return compactedSegments;
   }
 
-  public CompactionStatistics totalSkippedStatistics()
+  @Override
+  public List<CompactionCandidate> getSkippedSegments()
   {
-    return skippedSegmentStats;
+    return skippedSegments;
   }
 
   @Override
@@ -184,21 +199,13 @@ public class DataSourceCompactibleSegmentIterator implements Iterator<SegmentsTo
   }
 
   @Override
-  public SegmentsToCompact next()
+  public CompactionCandidate next()
   {
-    if (!hasNext()) {
+    if (hasNext()) {
+      return queue.poll();
+    } else {
       throw new NoSuchElementException();
     }
-
-    final SegmentsToCompact entry = queue.poll();
-    if (entry == null) {
-      throw new NoSuchElementException();
-    }
-
-    final List<DataSegment> resultSegments = entry.getSegments();
-    Preconditions.checkState(!resultSegments.isEmpty(), "Queue entry must not be empty");
-
-    return entry;
   }
 
   /**
@@ -296,96 +303,89 @@ public class DataSourceCompactibleSegmentIterator implements Iterator<SegmentsTo
    */
   private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compactibleSegmentIterator)
   {
-    final long inputSegmentSize = config.getInputSegmentSizeBytes();
     while (compactibleSegmentIterator.hasNext()) {
       List<DataSegment> segments = compactibleSegmentIterator.next();
+      if (CollectionUtils.isNullOrEmpty(segments)) {
+        continue;
+      }
 
-      // Do not compact an interval which comprises of a single tombstone
+      // Do not compact an interval which contains a single tombstone
       // If there are multiple tombstones in the interval, we may still want to compact them
       if (segments.size() == 1 && segments.get(0).isTombstone()) {
         continue;
       }
 
-      final SegmentsToCompact candidates = SegmentsToCompact.from(segments);
-      final Interval interval = candidates.getUmbrellaInterval();
-
-      final CompactionStatus compactionStatus = CompactionStatus.of(candidates, config, objectMapper);
-      if (!compactionStatus.isComplete()) {
-        log.debug(
-            "Datasource[%s], interval[%s] has [%d] segments that need to be compacted because [%s].",
-            dataSource, interval, candidates.size(), compactionStatus.getReasonToCompact()
-        );
-      }
+      final CompactionCandidate candidates = CompactionCandidate.from(segments);
+      final CompactionStatus compactionStatus
+          = statusTracker.computeCompactionStatus(candidates, config, searchPolicy);
+      final CompactionCandidate candidatesWithStatus = candidates.withCurrentStatus(compactionStatus);
+      statusTracker.onCompactionStatusComputed(candidatesWithStatus, config);
 
       if (compactionStatus.isComplete()) {
-        compactedSegmentStats.increment(candidates.getStats());
-      } else if (candidates.getTotalBytes() > inputSegmentSize) {
-        skippedSegmentStats.increment(candidates.getStats());
-        log.warn(
-            "Skipping compaction for datasource[%s], interval[%s] as total segment size[%d]"
-            + " is larger than allowed inputSegmentSize[%d].",
-            dataSource, interval, candidates.getTotalBytes(), inputSegmentSize
-        );
-      } else if (config.getGranularitySpec() != null
-                 && config.getGranularitySpec().getSegmentGranularity() != null) {
-        if (compactedIntervals.contains(interval)) {
-          // Skip these candidate segments as we have already compacted this interval
-        } else {
-          compactedIntervals.add(interval);
-          queue.add(candidates);
-        }
-      } else {
-        queue.add(candidates);
+        compactedSegments.add(candidatesWithStatus);
+      } else if (compactionStatus.isSkipped()) {
+        skippedSegments.add(candidatesWithStatus);
+      } else if (!queuedIntervals.contains(candidates.getUmbrellaInterval())) {
+        queue.add(candidatesWithStatus);
+        queuedIntervals.add(candidates.getUmbrellaInterval());
       }
     }
-
-    log.debug("No more segments to compact for datasource[%s].", dataSource);
   }
 
   /**
    * Returns the initial searchInterval which is {@code (timeline.first().start, timeline.last().end - skipOffset)}.
-   *
-   * @param timeline      timeline of a dataSource
-   * @param skipIntervals intervals to skip
-   *
-   * @return found interval to search or null if it's not found
    */
   private List<Interval> findInitialSearchInterval(
-      String dataSourceName,
       SegmentTimeline timeline,
-      Period skipOffset,
-      Granularity configuredSegmentGranularity,
       @Nullable List<Interval> skipIntervals
   )
   {
+    final Period skipOffset = config.getSkipOffsetFromLatest();
     Preconditions.checkArgument(timeline != null && !timeline.isEmpty(), "timeline should not be null or empty");
     Preconditions.checkNotNull(skipOffset, "skipOffset");
 
     final TimelineObjectHolder<String, DataSegment> first = Preconditions.checkNotNull(timeline.first(), "first");
     final TimelineObjectHolder<String, DataSegment> last = Preconditions.checkNotNull(timeline.last(), "last");
-    final List<Interval> fullSkipIntervals = sortAndAddSkipIntervalFromLatest(
+    final Interval latestSkipInterval = computeLatestSkipInterval(
+        config.getSegmentGranularity(),
         last.getInterval().getEnd(),
-        skipOffset,
-        configuredSegmentGranularity,
-        skipIntervals
+        skipOffset
     );
+    final List<Interval> allSkipIntervals
+        = sortAndAddSkipIntervalFromLatest(latestSkipInterval, skipIntervals);
 
     // Collect stats for all skipped segments
-    for (Interval skipInterval : fullSkipIntervals) {
+    for (Interval skipInterval : allSkipIntervals) {
       final List<DataSegment> segments = new ArrayList<>(
           timeline.findNonOvershadowedObjectsInInterval(skipInterval, Partitions.ONLY_COMPLETE)
       );
-      skippedSegmentStats.increment(SegmentsToCompact.from(segments).getStats());
+      if (!CollectionUtils.isNullOrEmpty(segments)) {
+        final CompactionCandidate candidates = CompactionCandidate.from(segments);
+
+        final CompactionStatus reason;
+        if (candidates.getUmbrellaInterval().overlaps(latestSkipInterval)) {
+          reason = CompactionStatus.skipped("skip offset from latest[%s]", skipOffset);
+        } else {
+          reason = CompactionStatus.skipped("interval locked by another task");
+        }
+
+        final CompactionCandidate candidatesWithStatus = candidates.withCurrentStatus(reason);
+        skippedSegments.add(candidatesWithStatus);
+        statusTracker.onCompactionStatusComputed(candidatesWithStatus, config);
+      }
     }
 
     final Interval totalInterval = new Interval(first.getInterval().getStart(), last.getInterval().getEnd());
-    final List<Interval> filteredInterval = filterSkipIntervals(totalInterval, fullSkipIntervals);
+    final List<Interval> filteredInterval = filterSkipIntervals(totalInterval, allSkipIntervals);
     final List<Interval> searchIntervals = new ArrayList<>();
 
     for (Interval lookupInterval : filteredInterval) {
       if (Intervals.ETERNITY.getStart().equals(lookupInterval.getStart())
           || Intervals.ETERNITY.getEnd().equals(lookupInterval.getEnd())) {
-        log.warn("Cannot compact datasource[%s] since interval[%s] coincides with ETERNITY.", dataSourceName, lookupInterval);
+        log.warn(
+            "Cannot compact datasource[%s] since interval[%s] coincides with ETERNITY.",
+            dataSource, lookupInterval
+        );
         return Collections.emptyList();
       }
       final List<DataSegment> segments = timeline
@@ -416,25 +416,30 @@ public class DataSourceCompactibleSegmentIterator implements Iterator<SegmentsTo
     return searchIntervals;
   }
 
+  static Interval computeLatestSkipInterval(
+      @Nullable Granularity configuredSegmentGranularity,
+      DateTime latestDataTimestamp,
+      Period skipOffsetFromLatest
+  )
+  {
+    if (configuredSegmentGranularity == null) {
+      return new Interval(skipOffsetFromLatest, latestDataTimestamp);
+    } else {
+      DateTime skipFromLastest = new DateTime(latestDataTimestamp, latestDataTimestamp.getZone()).minus(skipOffsetFromLatest);
+      DateTime skipOffsetBucketToSegmentGranularity = configuredSegmentGranularity.bucketStart(skipFromLastest);
+      return new Interval(skipOffsetBucketToSegmentGranularity, latestDataTimestamp);
+    }
+  }
+
   @VisibleForTesting
   static List<Interval> sortAndAddSkipIntervalFromLatest(
-      DateTime latest,
-      Period skipOffset,
-      Granularity configuredSegmentGranularity,
+      Interval skipFromLatest,
       @Nullable List<Interval> skipIntervals
   )
   {
     final List<Interval> nonNullSkipIntervals = skipIntervals == null
                                                 ? new ArrayList<>(1)
                                                 : new ArrayList<>(skipIntervals.size());
-    final Interval skipFromLatest;
-    if (configuredSegmentGranularity != null) {
-      DateTime skipFromLastest = new DateTime(latest, latest.getZone()).minus(skipOffset);
-      DateTime skipOffsetBucketToSegmentGranularity = configuredSegmentGranularity.bucketStart(skipFromLastest);
-      skipFromLatest = new Interval(skipOffsetBucketToSegmentGranularity, latest);
-    } else {
-      skipFromLatest = new Interval(skipOffset, latest);
-    }
 
     if (skipIntervals != null) {
       final List<Interval> sortedSkipIntervals = new ArrayList<>(skipIntervals);
diff --git a/server/src/main/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicy.java
new file mode 100644
index 00000000000..7c440900ba7
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicy.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.druid.java.util.common.guava.Comparators;
+
+import javax.annotation.Nullable;
+import java.util.Comparator;
+
+/**
+ * Implementation of {@link CompactionCandidateSearchPolicy} that prioritizes
+ * intervals which have the latest data.
+ */
+public class NewestSegmentFirstPolicy extends BaseCandidateSearchPolicy
+{
+  @JsonCreator
+  public NewestSegmentFirstPolicy(
+      @JsonProperty("priorityDatasource") @Nullable String priorityDatasource
+  )
+  {
+    super(priorityDatasource);
+  }
+
+  @Override
+  protected Comparator<CompactionCandidate> getSegmentComparator()
+  {
+    return (o1, o2) -> Comparators.intervalsByStartThenEnd()
+                                  .compare(o2.getUmbrellaInterval(), o1.getUmbrellaInterval());
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/compact/PriorityBasedCompactionSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/PriorityBasedCompactionSegmentIterator.java
similarity index 70%
rename from server/src/main/java/org/apache/druid/server/coordinator/compact/PriorityBasedCompactionSegmentIterator.java
rename to server/src/main/java/org/apache/druid/server/compaction/PriorityBasedCompactionSegmentIterator.java
index 33aea2a0451..a64c3b1e641 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/compact/PriorityBasedCompactionSegmentIterator.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/PriorityBasedCompactionSegmentIterator.java
@@ -17,45 +17,42 @@
  * under the License.
  */
 
-package org.apache.druid.server.coordinator.compact;
+package org.apache.druid.server.compaction;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
 import org.apache.druid.timeline.SegmentTimeline;
-import org.apache.druid.utils.CollectionUtils;
 import org.joda.time.Interval;
 
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.PriorityQueue;
+import java.util.stream.Collectors;
 
 /**
- * Implementation of {@link CompactionSegmentIterator} that returns segments in
- * order of their priority.
+ * Implementation of {@link CompactionSegmentIterator} that returns candidate
+ * segments in order of their priority.
  */
 public class PriorityBasedCompactionSegmentIterator implements CompactionSegmentIterator
 {
   private static final Logger log = new Logger(PriorityBasedCompactionSegmentIterator.class);
 
-  private final PriorityQueue<SegmentsToCompact> queue;
+  private final PriorityQueue<CompactionCandidate> queue;
   private final Map<String, DataSourceCompactibleSegmentIterator> datasourceIterators;
 
   public PriorityBasedCompactionSegmentIterator(
+      CompactionCandidateSearchPolicy searchPolicy,
       Map<String, DataSourceCompactionConfig> compactionConfigs,
       Map<String, SegmentTimeline> datasourceToTimeline,
       Map<String, List<Interval>> skipIntervals,
-      Comparator<SegmentsToCompact> segmentPriority,
-      ObjectMapper objectMapper
+      CompactionStatusTracker statusTracker
   )
   {
-    this.queue = new PriorityQueue<>(segmentPriority);
+    this.queue = new PriorityQueue<>(searchPolicy);
     this.datasourceIterators = Maps.newHashMapWithExpectedSize(datasourceToTimeline.size());
     compactionConfigs.forEach((datasource, config) -> {
       if (config == null) {
@@ -73,8 +70,8 @@ public class PriorityBasedCompactionSegmentIterator implements CompactionSegment
               compactionConfigs.get(datasource),
               timeline,
               skipIntervals.getOrDefault(datasource, Collections.emptyList()),
-              segmentPriority,
-              objectMapper
+              searchPolicy,
+              statusTracker
           )
       );
       addNextItemForDatasourceToQueue(datasource);
@@ -82,21 +79,19 @@ public class PriorityBasedCompactionSegmentIterator implements CompactionSegment
   }
 
   @Override
-  public Map<String, CompactionStatistics> totalCompactedStatistics()
+  public List<CompactionCandidate> getCompactedSegments()
   {
-    return CollectionUtils.mapValues(
-        datasourceIterators,
-        DataSourceCompactibleSegmentIterator::totalCompactedStatistics
-    );
+    return datasourceIterators.values().stream().flatMap(
+        iterator -> iterator.getCompactedSegments().stream()
+    ).collect(Collectors.toList());
   }
 
   @Override
-  public Map<String, CompactionStatistics> totalSkippedStatistics()
+  public List<CompactionCandidate> getSkippedSegments()
   {
-    return CollectionUtils.mapValues(
-        datasourceIterators,
-        DataSourceCompactibleSegmentIterator::totalSkippedStatistics
-    );
+    return datasourceIterators.values().stream().flatMap(
+        iterator -> iterator.getSkippedSegments().stream()
+    ).collect(Collectors.toList());
   }
 
   @Override
@@ -106,19 +101,18 @@ public class PriorityBasedCompactionSegmentIterator implements CompactionSegment
   }
 
   @Override
-  public SegmentsToCompact next()
+  public CompactionCandidate next()
   {
     if (!hasNext()) {
       throw new NoSuchElementException();
     }
 
-    final SegmentsToCompact entry = queue.poll();
+    final CompactionCandidate entry = queue.poll();
     if (entry == null) {
       throw new NoSuchElementException();
     }
-    Preconditions.checkState(!entry.isEmpty(), "Queue entry must not be empty");
 
-    addNextItemForDatasourceToQueue(entry.getFirst().getDataSource());
+    addNextItemForDatasourceToQueue(entry.getDataSource());
     return entry;
   }
 
@@ -126,9 +120,9 @@ public class PriorityBasedCompactionSegmentIterator implements CompactionSegment
   {
     final DataSourceCompactibleSegmentIterator iterator = datasourceIterators.get(dataSourceName);
     if (iterator.hasNext()) {
-      final SegmentsToCompact segmentsToCompact = iterator.next();
-      if (!segmentsToCompact.isEmpty()) {
-        queue.add(segmentsToCompact);
+      final CompactionCandidate compactionCandidate = iterator.next();
+      if (compactionCandidate != null) {
+        queue.add(compactionCandidate);
       }
     }
   }
diff --git a/server/src/main/java/org/apache/druid/server/compaction/Table.java b/server/src/main/java/org/apache/druid/server/compaction/Table.java
new file mode 100644
index 00000000000..5a620dacc8d
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/Table.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A simple table POJO with any number of rows and specified column names.
+ * Used in {@link CompactionSimulateResult}.
+ */
+public class Table
+{
+  private final List<String> columnNames;
+  private final List<List<Object>> rows = new ArrayList<>();
+
+  public static Table withColumnNames(String... columnNames)
+  {
+    return new Table(Arrays.asList(columnNames), null);
+  }
+
+  @JsonCreator
+  public Table(
+      @JsonProperty("columnNames") List<String> columnNames,
+      @JsonProperty("rows") List<List<Object>> rows
+  )
+  {
+    this.columnNames = columnNames;
+    if (rows != null) {
+      this.rows.addAll(rows);
+    }
+  }
+
+  @JsonProperty
+  public List<String> getColumnNames()
+  {
+    return columnNames;
+  }
+
+  @JsonProperty
+  public List<List<Object>> getRows()
+  {
+    return rows;
+  }
+
+  public void addRow(Object... values)
+  {
+    rows.add(Arrays.asList(values));
+  }
+
+  public boolean isEmpty()
+  {
+    return rows.isEmpty();
+  }
+
+  @Override
+  public String toString()
+  {
+    return "Table{" +
+           "columnNames=" + columnNames +
+           ", rows=" + rows +
+           '}';
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/AutoCompactionSnapshot.java b/server/src/main/java/org/apache/druid/server/coordinator/AutoCompactionSnapshot.java
index d52d4e9eba0..d6fa4835b48 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/AutoCompactionSnapshot.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/AutoCompactionSnapshot.java
@@ -21,8 +21,9 @@ package org.apache.druid.server.coordinator;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
 import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.server.coordinator.compact.CompactionStatistics;
+import org.apache.druid.server.compaction.CompactionStatistics;
 
 import javax.validation.constraints.NotNull;
 import java.util.Objects;
@@ -60,22 +61,22 @@ public class AutoCompactionSnapshot
 
   public static Builder builder(String dataSource)
   {
-    return new Builder(dataSource, AutoCompactionScheduleStatus.RUNNING);
+    return new Builder(dataSource).withStatus(AutoCompactionScheduleStatus.RUNNING);
   }
 
   @JsonCreator
   public AutoCompactionSnapshot(
-      @JsonProperty @NotNull String dataSource,
-      @JsonProperty @NotNull AutoCompactionScheduleStatus scheduleStatus,
-      @JsonProperty long bytesAwaitingCompaction,
-      @JsonProperty long bytesCompacted,
-      @JsonProperty long bytesSkipped,
-      @JsonProperty long segmentCountAwaitingCompaction,
-      @JsonProperty long segmentCountCompacted,
-      @JsonProperty long segmentCountSkipped,
-      @JsonProperty long intervalCountAwaitingCompaction,
-      @JsonProperty long intervalCountCompacted,
-      @JsonProperty long intervalCountSkipped
+      @JsonProperty("dataSource") @NotNull String dataSource,
+      @JsonProperty("scheduleStatus") @NotNull AutoCompactionScheduleStatus scheduleStatus,
+      @JsonProperty("bytesAwaitingCompaction") long bytesAwaitingCompaction,
+      @JsonProperty("bytesCompacted") long bytesCompacted,
+      @JsonProperty("bytesSkipped") long bytesSkipped,
+      @JsonProperty("segmentCountAwaitingCompaction") long segmentCountAwaitingCompaction,
+      @JsonProperty("segmentCountCompacted") long segmentCountCompacted,
+      @JsonProperty("segmentCountSkipped") long segmentCountSkipped,
+      @JsonProperty("intervalCountAwaitingCompaction") long intervalCountAwaitingCompaction,
+      @JsonProperty("intervalCountCompacted") long intervalCountCompacted,
+      @JsonProperty("intervalCountSkipped") long intervalCountSkipped
   )
   {
     this.dataSource = dataSource;
@@ -192,26 +193,26 @@ public class AutoCompactionSnapshot
   public static class Builder
   {
     private final String dataSource;
-    private final AutoCompactionScheduleStatus scheduleStatus;
+    private AutoCompactionScheduleStatus scheduleStatus;
 
     private final CompactionStatistics compactedStats = new CompactionStatistics();
     private final CompactionStatistics skippedStats = new CompactionStatistics();
     private final CompactionStatistics waitingStats = new CompactionStatistics();
 
     private Builder(
-        @NotNull String dataSource,
-        @NotNull AutoCompactionScheduleStatus scheduleStatus
+        @NotNull String dataSource
     )
     {
       if (dataSource == null || dataSource.isEmpty()) {
         throw new ISE("Invalid dataSource name");
       }
-      if (scheduleStatus == null) {
-        throw new ISE("scheduleStatus cannot be null");
-      }
-
       this.dataSource = dataSource;
-      this.scheduleStatus = scheduleStatus;
+    }
+
+    public Builder withStatus(AutoCompactionScheduleStatus status)
+    {
+      this.scheduleStatus = Preconditions.checkNotNull(status, "scheduleStatus cannot be null");
+      return this;
     }
 
     public void incrementWaitingStats(CompactionStatistics entry)
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/ClusterCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/ClusterCompactionConfig.java
index 6009dc12cf4..e2b98a32a92 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/ClusterCompactionConfig.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/ClusterCompactionConfig.java
@@ -22,6 +22,7 @@ package org.apache.druid.server.coordinator;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.druid.indexer.CompactionEngine;
+import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy;
 
 import javax.annotation.Nullable;
 import java.util.Objects;
@@ -37,18 +38,21 @@ public class ClusterCompactionConfig
   private final Integer maxCompactionTaskSlots;
   private final Boolean useAutoScaleSlots;
   private final CompactionEngine engine;
+  private final CompactionCandidateSearchPolicy compactionPolicy;
 
   @JsonCreator
   public ClusterCompactionConfig(
       @JsonProperty("compactionTaskSlotRatio") @Nullable Double compactionTaskSlotRatio,
       @JsonProperty("maxCompactionTaskSlots") @Nullable Integer maxCompactionTaskSlots,
       @JsonProperty("useAutoScaleSlots") @Nullable Boolean useAutoScaleSlots,
-      @JsonProperty("engine") @Nullable CompactionEngine engine
+      @JsonProperty("engine") @Nullable CompactionEngine engine,
+      @JsonProperty("compactionPolicy") @Nullable CompactionCandidateSearchPolicy compactionPolicy
   )
   {
     this.compactionTaskSlotRatio = compactionTaskSlotRatio;
     this.maxCompactionTaskSlots = maxCompactionTaskSlots;
     this.useAutoScaleSlots = useAutoScaleSlots;
+    this.compactionPolicy = compactionPolicy;
     this.engine = engine;
   }
 
@@ -80,6 +84,13 @@ public class ClusterCompactionConfig
     return engine;
   }
 
+  @Nullable
+  @JsonProperty
+  public CompactionCandidateSearchPolicy getCompactionPolicy()
+  {
+    return compactionPolicy;
+  }
+
   @Override
   public boolean equals(Object o)
   {
@@ -93,12 +104,19 @@ public class ClusterCompactionConfig
     return Objects.equals(compactionTaskSlotRatio, that.compactionTaskSlotRatio)
            && Objects.equals(maxCompactionTaskSlots, that.maxCompactionTaskSlots)
            && Objects.equals(useAutoScaleSlots, that.useAutoScaleSlots)
+           && Objects.equals(compactionPolicy, that.compactionPolicy)
            && engine == that.engine;
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(compactionTaskSlotRatio, maxCompactionTaskSlots, useAutoScaleSlots, engine);
+    return Objects.hash(
+        compactionTaskSlotRatio,
+        maxCompactionTaskSlots,
+        useAutoScaleSlots,
+        compactionPolicy,
+        engine
+    );
   }
 }
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CompactionSupervisorConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/CompactionSupervisorConfig.java
new file mode 100644
index 00000000000..e738a3e7f0b
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/coordinator/CompactionSupervisorConfig.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.coordinator;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.druid.common.config.Configs;
+
+import javax.annotation.Nullable;
+import java.util.Objects;
+
+/**
+ * This config must be bound on CliOverlord to enable running compaction supervisors
+ * on the Overlord. When compaction supervisors are enabled, the Coordinator
+ * does not run auto-compact duty.
+ */
+public class CompactionSupervisorConfig
+{
+  private static final CompactionSupervisorConfig DEFAULT = new CompactionSupervisorConfig(null);
+
+  @JsonProperty
+  private final boolean enabled;
+
+  public static CompactionSupervisorConfig defaultConfig()
+  {
+    return DEFAULT;
+  }
+
+  @JsonCreator
+  public CompactionSupervisorConfig(
+      @JsonProperty("enabled") @Nullable Boolean enabled
+  )
+  {
+    this.enabled = Configs.valueOrDefault(enabled, false);
+  }
+
+  public boolean isEnabled()
+  {
+    return enabled;
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    CompactionSupervisorConfig that = (CompactionSupervisorConfig) o;
+    return enabled == that.enabled;
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hashCode(enabled);
+  }
+
+  @Override
+  public String toString()
+  {
+    return "CompactionSchedulerConfig{" +
+           "enabled=" + enabled +
+           '}';
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java
index 193b53a5d05..b4b2be78032 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java
@@ -20,9 +20,11 @@
 package org.apache.druid.server.coordinator;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
 import org.apache.druid.indexer.CompactionEngine;
+import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.joda.time.Period;
 
@@ -188,6 +190,13 @@ public class DataSourceCompactionConfig
     return engine;
   }
 
+  @Nullable
+  @JsonIgnore
+  public Granularity getSegmentGranularity()
+  {
+    return granularitySpec == null ? null : granularitySpec.getSegmentGranularity();
+  }
+
   @Override
   public boolean equals(Object o)
   {
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCompactionConfig.java
index 7793b55c4b9..b35ba7d2938 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCompactionConfig.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCompactionConfig.java
@@ -24,6 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Optional;
 import org.apache.druid.common.config.Configs;
 import org.apache.druid.indexer.CompactionEngine;
+import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy;
+import org.apache.druid.server.compaction.NewestSegmentFirstPolicy;
 
 import javax.annotation.Nullable;
 import java.util.ArrayList;
@@ -38,14 +40,17 @@ public class DruidCompactionConfig
 {
   public static final String CONFIG_KEY = "coordinator.compaction.config";
 
+  private static final CompactionCandidateSearchPolicy DEFAULT_COMPACTION_POLICY
+      = new NewestSegmentFirstPolicy(null);
   private static final DruidCompactionConfig EMPTY_INSTANCE
-      = new DruidCompactionConfig(Collections.emptyList(), null, null, null, null);
+      = new DruidCompactionConfig(Collections.emptyList(), null, null, null, null, null);
 
   private final List<DataSourceCompactionConfig> compactionConfigs;
   private final double compactionTaskSlotRatio;
   private final int maxCompactionTaskSlots;
   private final boolean useAutoScaleSlots;
   private final CompactionEngine engine;
+  private final CompactionCandidateSearchPolicy compactionPolicy;
 
   public DruidCompactionConfig withDatasourceConfigs(
       List<DataSourceCompactionConfig> compactionConfigs
@@ -56,7 +61,8 @@ public class DruidCompactionConfig
         compactionTaskSlotRatio,
         maxCompactionTaskSlots,
         useAutoScaleSlots,
-        engine
+        engine,
+        compactionPolicy
     );
   }
 
@@ -69,7 +75,8 @@ public class DruidCompactionConfig
         Configs.valueOrDefault(update.getCompactionTaskSlotRatio(), compactionTaskSlotRatio),
         Configs.valueOrDefault(update.getMaxCompactionTaskSlots(), maxCompactionTaskSlots),
         Configs.valueOrDefault(update.getUseAutoScaleSlots(), useAutoScaleSlots),
-        Configs.valueOrDefault(update.getEngine(), engine)
+        Configs.valueOrDefault(update.getEngine(), engine),
+        Configs.valueOrDefault(update.getCompactionPolicy(), compactionPolicy)
     );
   }
 
@@ -91,7 +98,8 @@ public class DruidCompactionConfig
       @JsonProperty("compactionTaskSlotRatio") @Nullable Double compactionTaskSlotRatio,
       @JsonProperty("maxCompactionTaskSlots") @Nullable Integer maxCompactionTaskSlots,
       @JsonProperty("useAutoScaleSlots") @Nullable Boolean useAutoScaleSlots,
-      @JsonProperty("engine") @Nullable CompactionEngine compactionEngine
+      @JsonProperty("engine") @Nullable CompactionEngine compactionEngine,
+      @JsonProperty("compactionPolicy") @Nullable CompactionCandidateSearchPolicy compactionPolicy
   )
   {
     this.compactionConfigs = Configs.valueOrDefault(compactionConfigs, Collections.emptyList());
@@ -99,6 +107,7 @@ public class DruidCompactionConfig
     this.maxCompactionTaskSlots = Configs.valueOrDefault(maxCompactionTaskSlots, Integer.MAX_VALUE);
     this.useAutoScaleSlots = Configs.valueOrDefault(useAutoScaleSlots, false);
     this.engine = Configs.valueOrDefault(compactionEngine, CompactionEngine.NATIVE);
+    this.compactionPolicy = Configs.valueOrDefault(compactionPolicy, DEFAULT_COMPACTION_POLICY);
   }
 
   @JsonProperty
@@ -139,7 +148,8 @@ public class DruidCompactionConfig
         compactionTaskSlotRatio,
         maxCompactionTaskSlots,
         useAutoScaleSlots,
-        engine
+        engine,
+        compactionPolicy
     );
   }
 
@@ -160,6 +170,12 @@ public class DruidCompactionConfig
     return Optional.absent();
   }
 
+  @JsonProperty
+  public CompactionCandidateSearchPolicy getCompactionPolicy()
+  {
+    return compactionPolicy;
+  }
+
   @Override
   public boolean equals(Object o)
   {
@@ -174,6 +190,7 @@ public class DruidCompactionConfig
            maxCompactionTaskSlots == that.maxCompactionTaskSlots &&
            useAutoScaleSlots == that.useAutoScaleSlots &&
            engine == that.engine &&
+           Objects.equals(compactionPolicy, that.compactionPolicy) &&
            Objects.equals(compactionConfigs, that.compactionConfigs);
   }
 
@@ -185,7 +202,8 @@ public class DruidCompactionConfig
         compactionTaskSlotRatio,
         maxCompactionTaskSlots,
         useAutoScaleSlots,
-        engine
+        engine,
+        compactionPolicy
     );
   }
 
@@ -198,6 +216,7 @@ public class DruidCompactionConfig
            ", maxCompactionTaskSlots=" + maxCompactionTaskSlots +
            ", useAutoScaleSlots=" + useAutoScaleSlots +
            ", engine=" + engine +
+           ", compactionPolicy=" + compactionPolicy +
            '}';
   }
 }
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
index 5e468182fa7..6f907354f08 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
@@ -35,6 +35,7 @@ import org.apache.druid.client.DruidServer;
 import org.apache.druid.client.ImmutableDruidDataSource;
 import org.apache.druid.client.ServerInventoryView;
 import org.apache.druid.client.coordinator.Coordinator;
+import org.apache.druid.common.guava.FutureUtils;
 import org.apache.druid.curator.discovery.ServiceAnnouncer;
 import org.apache.druid.discovery.DruidLeaderSelector;
 import org.apache.druid.guice.ManageLifecycle;
@@ -54,8 +55,10 @@ import org.apache.druid.rpc.indexing.OverlordClient;
 import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
 import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache;
 import org.apache.druid.server.DruidNode;
+import org.apache.druid.server.compaction.CompactionRunSimulator;
+import org.apache.druid.server.compaction.CompactionSimulateResult;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
 import org.apache.druid.server.coordinator.balancer.BalancerStrategyFactory;
-import org.apache.druid.server.coordinator.compact.CompactionSegmentSearchPolicy;
 import org.apache.druid.server.coordinator.config.CoordinatorKillConfigs;
 import org.apache.druid.server.coordinator.config.DruidCoordinatorConfig;
 import org.apache.druid.server.coordinator.config.KillUnusedSegmentsConfig;
@@ -153,12 +156,12 @@ public class DruidCoordinator
   private final BalancerStrategyFactory balancerStrategyFactory;
   private final LookupCoordinatorManager lookupCoordinatorManager;
   private final DruidLeaderSelector coordLeaderSelector;
+  private final CompactionStatusTracker compactionStatusTracker;
   private final CompactSegments compactSegments;
   @Nullable
   private final CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache;
   private final CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig;
 
-
   private volatile boolean started = false;
 
   /**
@@ -200,9 +203,9 @@ public class DruidCoordinator
       CoordinatorCustomDutyGroups customDutyGroups,
       LookupCoordinatorManager lookupCoordinatorManager,
       @Coordinator DruidLeaderSelector coordLeaderSelector,
-      CompactionSegmentSearchPolicy compactionSegmentSearchPolicy,
       @Nullable CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache,
-      CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig
+      CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig,
+      CompactionStatusTracker compactionStatusTracker
   )
   {
     this.config = config;
@@ -220,7 +223,8 @@ public class DruidCoordinator
     this.balancerStrategyFactory = config.getBalancerStrategyFactory();
     this.lookupCoordinatorManager = lookupCoordinatorManager;
     this.coordLeaderSelector = coordLeaderSelector;
-    this.compactSegments = initializeCompactSegmentsDuty(compactionSegmentSearchPolicy);
+    this.compactionStatusTracker = compactionStatusTracker;
+    this.compactSegments = initializeCompactSegmentsDuty(this.compactionStatusTracker);
     this.loadQueueManager = loadQueueManager;
     this.coordinatorSegmentMetadataCache = coordinatorSegmentMetadataCache;
     this.centralizedDatasourceSchemaConfig = centralizedDatasourceSchemaConfig;
@@ -341,12 +345,6 @@ public class DruidCoordinator
     return replicaCountsInCluster == null ? null : replicaCountsInCluster.required();
   }
 
-  @Nullable
-  public Long getTotalSizeOfSegmentsAwaitingCompaction(String dataSource)
-  {
-    return compactSegments.getTotalSizeOfSegmentsAwaitingCompaction(dataSource);
-  }
-
   @Nullable
   public AutoCompactionSnapshot getAutoCompactionSnapshotForDataSource(String dataSource)
   {
@@ -358,6 +356,16 @@ public class DruidCoordinator
     return compactSegments.getAutoCompactionSnapshot();
   }
 
+  public CompactionSimulateResult simulateRunWithConfigUpdate(ClusterCompactionConfig updateRequest)
+  {
+    return new CompactionRunSimulator(compactionStatusTracker, overlordClient).simulateRunWithConfig(
+        metadataManager.configs().getCurrentCompactionConfig().withClusterConfig(updateRequest),
+        metadataManager.segments()
+                       .getSnapshotOfDataSourcesWithAllUsedSegments()
+                       .getUsedSegmentsTimelinesPerDataSource()
+    );
+  }
+
   public String getCurrentLeader()
   {
     return coordLeaderSelector.getCurrentLeader();
@@ -533,6 +541,7 @@ public class DruidCoordinator
       if (coordinatorSegmentMetadataCache != null) {
         coordinatorSegmentMetadataCache.onLeaderStop();
       }
+      compactionStatusTracker.stop();
       taskMaster.onLeaderStop();
       serviceAnnouncer.unannounce(self);
       lookupCoordinatorManager.stop();
@@ -541,6 +550,20 @@ public class DruidCoordinator
     }
   }
 
+  /**
+   * Check if compaction supervisors are enabled on the Overlord.
+   */
+  private boolean isCompactionSupervisorEnabled()
+  {
+    try {
+      return FutureUtils.getUnchecked(overlordClient.isCompactionSupervisorEnabled(), true);
+    }
+    catch (Exception e) {
+      // The Overlord is probably on an older version, assume that compaction supervisor is not enabled
+      return false;
+    }
+  }
+
   @GuardedBy("lock")
   private ScheduledExecutorService getOrCreateDutyGroupExecutor(String dutyGroup)
   {
@@ -590,8 +613,7 @@ public class DruidCoordinator
       duties.add(new KillStalePendingSegments(overlordClient));
     }
 
-    // CompactSegmentsDuty should be the last duty as it can take a long time to complete
-    // We do not have to add compactSegments if it is already enabled in the custom duty group
+    // Do not add compactSegments if it is already included in the custom duty groups
     if (getCompactSegmentsDutyFromCustomGroups().isEmpty()) {
       duties.add(compactSegments);
     }
@@ -625,11 +647,11 @@ public class DruidCoordinator
   }
 
   @VisibleForTesting
-  CompactSegments initializeCompactSegmentsDuty(CompactionSegmentSearchPolicy compactionSegmentSearchPolicy)
+  CompactSegments initializeCompactSegmentsDuty(CompactionStatusTracker statusTracker)
   {
     List<CompactSegments> compactSegmentsDutyFromCustomGroups = getCompactSegmentsDutyFromCustomGroups();
     if (compactSegmentsDutyFromCustomGroups.isEmpty()) {
-      return new CompactSegments(compactionSegmentSearchPolicy, overlordClient);
+      return new CompactSegments(statusTracker, overlordClient);
     } else {
       if (compactSegmentsDutyFromCustomGroups.size() > 1) {
         log.warn(
@@ -735,6 +757,10 @@ public class DruidCoordinator
               && coordLeaderSelector.isLeader()
               && startingLeaderCounter == coordLeaderSelector.localTerm()) {
 
+            if (shouldSkipAutoCompactDuty(duty)) {
+              continue;
+            }
+
             dutyRunTime.restart();
             params = duty.run(params);
             dutyRunTime.stop();
@@ -781,6 +807,26 @@ public class DruidCoordinator
       }
     }
 
+    /**
+     * @return true if this is an auto-compact CompactSegments duty and should
+     * not be run in case Compaction Scheduler is already running on Overlord.
+     * Manually triggered compaction should always be run.
+     */
+    private boolean shouldSkipAutoCompactDuty(CoordinatorDuty duty)
+    {
+      final boolean shouldSkipDuty = duty instanceof CompactSegments
+                                     && !COMPACT_SEGMENTS_DUTIES_DUTY_GROUP.equals(dutyGroupName)
+                                     && isCompactionSupervisorEnabled();
+      if (shouldSkipDuty) {
+        log.warn(
+            "Skipping Compact Segments duty in group[%s] since compaction"
+            + " supervisors are already running on Overlord.",
+            dutyGroupName
+        );
+      }
+      return shouldSkipDuty;
+    }
+
     private void emitStat(CoordinatorStat stat, Map<Dimension, String> dimensionValues, long value)
     {
       ServiceMetricEvent.Builder eventBuilder = new ServiceMetricEvent.Builder()
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionSegmentIterator.java b/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionSegmentIterator.java
deleted file mode 100644
index bab7ca8f92f..00000000000
--- a/server/src/main/java/org/apache/druid/server/coordinator/compact/CompactionSegmentIterator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.server.coordinator.compact;
-
-import org.apache.druid.timeline.DataSegment;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * Segments in the lists which are the elements of this iterator are sorted according to the natural segment order
- * (see {@link DataSegment#compareTo}).
- */
-public interface CompactionSegmentIterator extends Iterator<SegmentsToCompact>
-{
-  /**
-   * Return a map of dataSourceName to CompactionStatistics.
-   * This method returns the aggregated statistics of segments that was already compacted and does not need to be compacted
-   * again. Hence, segment that were not returned by the {@link Iterator#next()} becuase it does not needs compaction.
-   * Note that the aggregations returned by this method is only up to the current point of the iterator being iterated.
-   */
-  Map<String, CompactionStatistics> totalCompactedStatistics();
-
-  /**
-   * Return a map of dataSourceName to CompactionStatistics.
-   * This method returns the aggregated statistics of segments that was skipped as it cannot be compacted.
-   * Hence, segment that were not returned by the {@link Iterator#next()} becuase it cannot be compacted.
-   * Note that the aggregations returned by this method is only up to the current point of the iterator being iterated.
-   */
-  Map<String, CompactionStatistics> totalSkippedStatistics();
-
-}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/compact/NewestSegmentFirstPolicy.java b/server/src/main/java/org/apache/druid/server/coordinator/compact/NewestSegmentFirstPolicy.java
deleted file mode 100644
index bc923da4f80..00000000000
--- a/server/src/main/java/org/apache/druid/server/coordinator/compact/NewestSegmentFirstPolicy.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.server.coordinator.compact;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.inject.Inject;
-import org.apache.druid.java.util.common.guava.Comparators;
-import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
-import org.apache.druid.timeline.SegmentTimeline;
-import org.joda.time.Interval;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * This policy searches segments for compaction from newest to oldest.
- */
-public class NewestSegmentFirstPolicy implements CompactionSegmentSearchPolicy
-{
-  private final ObjectMapper objectMapper;
-
-  @Inject
-  public NewestSegmentFirstPolicy(ObjectMapper objectMapper)
-  {
-    this.objectMapper = objectMapper;
-  }
-
-  @Override
-  public CompactionSegmentIterator createIterator(
-      Map<String, DataSourceCompactionConfig> compactionConfigs,
-      Map<String, SegmentTimeline> dataSources,
-      Map<String, List<Interval>> skipIntervals
-  )
-  {
-    return new PriorityBasedCompactionSegmentIterator(
-        compactionConfigs,
-        dataSources,
-        skipIntervals,
-        (o1, o2) -> Comparators.intervalsByStartThenEnd()
-                               .compare(o2.getUmbrellaInterval(), o1.getUmbrellaInterval()),
-        objectMapper
-    );
-  }
-
-}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java
index 7b2392b8c66..a2f97f298af 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java
@@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
-import com.google.inject.Inject;
 import org.apache.druid.client.indexing.ClientCompactionIOConfig;
 import org.apache.druid.client.indexing.ClientCompactionIntervalSpec;
 import org.apache.druid.client.indexing.ClientCompactionRunnerInfo;
@@ -34,11 +33,11 @@ import org.apache.druid.client.indexing.ClientCompactionTaskQuery;
 import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig;
 import org.apache.druid.client.indexing.ClientCompactionTaskTransformSpec;
 import org.apache.druid.client.indexing.ClientMSQContext;
-import org.apache.druid.client.indexing.ClientTaskQuery;
 import org.apache.druid.client.indexing.TaskPayloadResponse;
 import org.apache.druid.common.guava.FutureUtils;
 import org.apache.druid.common.utils.IdUtils;
 import org.apache.druid.indexer.CompactionEngine;
+import org.apache.druid.indexer.TaskStatus;
 import org.apache.druid.indexer.TaskStatusPlus;
 import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
 import org.apache.druid.java.util.common.ISE;
@@ -48,13 +47,15 @@ import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.server.compaction.CompactionCandidate;
+import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy;
+import org.apache.druid.server.compaction.CompactionSegmentIterator;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
+import org.apache.druid.server.compaction.PriorityBasedCompactionSegmentIterator;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
 import org.apache.druid.server.coordinator.DruidCompactionConfig;
 import org.apache.druid.server.coordinator.DruidCoordinatorRuntimeParams;
-import org.apache.druid.server.coordinator.compact.CompactionSegmentIterator;
-import org.apache.druid.server.coordinator.compact.CompactionSegmentSearchPolicy;
-import org.apache.druid.server.coordinator.compact.SegmentsToCompact;
 import org.apache.druid.server.coordinator.stats.CoordinatorRunStats;
 import org.apache.druid.server.coordinator.stats.Dimension;
 import org.apache.druid.server.coordinator.stats.RowKey;
@@ -67,8 +68,10 @@ import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -90,22 +93,21 @@ public class CompactSegments implements CoordinatorCustomDuty
   private static final Predicate<TaskStatusPlus> IS_COMPACTION_TASK =
       status -> null != status && COMPACTION_TASK_TYPE.equals(status.getType());
 
-  private final CompactionSegmentSearchPolicy policy;
+  private final CompactionStatusTracker statusTracker;
   private final OverlordClient overlordClient;
 
   // This variable is updated by the Coordinator thread executing duties and
   // read by HTTP threads processing Coordinator API calls.
   private final AtomicReference<Map<String, AutoCompactionSnapshot>> autoCompactionSnapshotPerDataSource = new AtomicReference<>();
 
-  @Inject
   @JsonCreator
   public CompactSegments(
-      @JacksonInject CompactionSegmentSearchPolicy policy,
+      @JacksonInject CompactionStatusTracker statusTracker,
       @JacksonInject OverlordClient overlordClient
   )
   {
-    this.policy = policy;
     this.overlordClient = overlordClient;
+    this.statusTracker = statusTracker;
     resetCompactionSnapshot();
   }
 
@@ -117,22 +119,36 @@ public class CompactSegments implements CoordinatorCustomDuty
 
   @Override
   public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params)
+  {
+    run(
+        params.getCompactionConfig(),
+        params.getUsedSegmentsTimelinesPerDataSource(),
+        params.getCoordinatorStats()
+    );
+    return params;
+  }
+
+  public void run(
+      DruidCompactionConfig dynamicConfig,
+      Map<String, SegmentTimeline> dataSources,
+      CoordinatorRunStats stats
+  )
   {
     LOG.info("Running CompactSegments duty");
 
-    final DruidCompactionConfig dynamicConfig = params.getCompactionConfig();
     final int maxCompactionTaskSlots = dynamicConfig.getMaxCompactionTaskSlots();
     if (maxCompactionTaskSlots <= 0) {
       LOG.info("Skipping compaction as maxCompactionTaskSlots is [%d].", maxCompactionTaskSlots);
       resetCompactionSnapshot();
-      return params;
+      return;
     }
 
+    statusTracker.onCompactionConfigUpdated(dynamicConfig);
     List<DataSourceCompactionConfig> compactionConfigList = dynamicConfig.getCompactionConfigs();
     if (compactionConfigList == null || compactionConfigList.isEmpty()) {
       LOG.info("Skipping compaction as compaction config list is empty.");
       resetCompactionSnapshot();
-      return params;
+      return;
     }
 
     Map<String, DataSourceCompactionConfig> compactionConfigs = compactionConfigList
@@ -144,10 +160,15 @@ public class CompactSegments implements CoordinatorCustomDuty
 
     // Fetch currently running compaction tasks
     int busyCompactionTaskSlots = 0;
-    final List<TaskStatusPlus> compactionTasks = CoordinatorDutyUtils.getNumActiveTaskSlots(
+    final List<TaskStatusPlus> compactionTasks = CoordinatorDutyUtils.getStatusOfActiveTasks(
         overlordClient,
         IS_COMPACTION_TASK
     );
+
+    final Set<String> activeTaskIds
+        = compactionTasks.stream().map(TaskStatusPlus::getId).collect(Collectors.toSet());
+    trackStatusOfCompletedTasks(activeTaskIds);
+
     for (TaskStatusPlus status : compactionTasks) {
       final TaskPayloadResponse response =
           FutureUtils.getUnchecked(overlordClient.taskPayload(status.getId()), true);
@@ -194,9 +215,14 @@ public class CompactSegments implements CoordinatorCustomDuty
     );
 
     // Get iterator over segments to compact and submit compaction tasks
-    Map<String, SegmentTimeline> dataSources = params.getUsedSegmentsTimelinesPerDataSource();
-    final CompactionSegmentIterator iterator =
-        policy.createIterator(compactionConfigs, dataSources, intervalsToSkipCompaction);
+    final CompactionCandidateSearchPolicy policy = dynamicConfig.getCompactionPolicy();
+    final CompactionSegmentIterator iterator = new PriorityBasedCompactionSegmentIterator(
+        policy,
+        compactionConfigs,
+        dataSources,
+        intervalsToSkipCompaction,
+        statusTracker
+    );
 
     final int compactionTaskCapacity = getCompactionTaskCapacity(dynamicConfig);
     final int availableCompactionTaskSlots
@@ -211,13 +237,10 @@ public class CompactSegments implements CoordinatorCustomDuty
         dynamicConfig.getEngine()
     );
 
-    final CoordinatorRunStats stats = params.getCoordinatorStats();
     stats.add(Stats.Compaction.MAX_SLOTS, compactionTaskCapacity);
     stats.add(Stats.Compaction.AVAILABLE_SLOTS, availableCompactionTaskSlots);
     stats.add(Stats.Compaction.SUBMITTED_TASKS, numSubmittedCompactionTasks);
     updateCompactionSnapshotStats(currentRunAutoCompactionSnapshotBuilders, iterator, stats);
-
-    return params;
   }
 
   private void resetCompactionSnapshot()
@@ -225,6 +248,31 @@ public class CompactSegments implements CoordinatorCustomDuty
     autoCompactionSnapshotPerDataSource.set(Collections.emptyMap());
   }
 
+  /**
+   * Queries the Overlord for the status of all tasks that were submitted
+   * recently but are not active anymore. The statuses are then updated in the
+   * {@link #statusTracker}.
+   */
+  private void trackStatusOfCompletedTasks(Set<String> activeTaskIds)
+  {
+    final Set<String> finishedTaskIds = new HashSet<>(statusTracker.getSubmittedTaskIds());
+    finishedTaskIds.removeAll(activeTaskIds);
+
+    if (finishedTaskIds.isEmpty()) {
+      return;
+    }
+
+    final Map<String, TaskStatus> taskStatusMap
+        = FutureUtils.getUnchecked(overlordClient.taskStatuses(finishedTaskIds), true);
+    for (String taskId : finishedTaskIds) {
+      // Assume unknown task to have finished successfully
+      final TaskStatus taskStatus = taskStatusMap.getOrDefault(taskId, TaskStatus.success(taskId));
+      if (taskStatus.isComplete()) {
+        statusTracker.onTaskFinished(taskId, taskStatus);
+      }
+    }
+  }
+
   /**
    * Cancels a currently running compaction task if the segment granularity
    * for this datasource has changed in the compaction config.
@@ -295,8 +343,7 @@ public class CompactSegments implements CoordinatorCustomDuty
    * Returns the maximum number of task slots used by one native compaction task at any time when the task is
    * issued with the given tuningConfig.
    */
-  @VisibleForTesting
-  static int findMaxNumTaskSlotsUsedByOneNativeCompactionTask(
+  public static int findMaxNumTaskSlotsUsedByOneNativeCompactionTask(
       @Nullable ClientCompactionTaskQueryTuningConfig tuningConfig
   )
   {
@@ -392,12 +439,8 @@ public class CompactSegments implements CoordinatorCustomDuty
     int totalTaskSlotsAssigned = 0;
 
     while (iterator.hasNext() && totalTaskSlotsAssigned < numAvailableCompactionTaskSlots) {
-      final SegmentsToCompact entry = iterator.next();
-      if (entry.isEmpty()) {
-        throw new ISE("segmentsToCompact is empty?");
-      }
-
-      final String dataSourceName = entry.getFirst().getDataSource();
+      final CompactionCandidate entry = iterator.next();
+      final String dataSourceName = entry.getDataSource();
 
       // As these segments will be compacted, we will aggregate the statistic to the Compacted statistics
       currentRunAutoCompactionSnapshotBuilders
@@ -408,7 +451,6 @@ public class CompactSegments implements CoordinatorCustomDuty
       final List<DataSegment> segmentsToCompact = entry.getSegments();
 
       // Create granularitySpec to send to compaction task
-      ClientCompactionTaskGranularitySpec granularitySpec;
       Granularity segmentGranularityToUse = null;
       if (config.getGranularitySpec() == null || config.getGranularitySpec().getSegmentGranularity() == null) {
         // Determines segmentGranularity from the segmentsToCompact
@@ -433,14 +475,14 @@ public class CompactSegments implements CoordinatorCustomDuty
       } else {
         segmentGranularityToUse = config.getGranularitySpec().getSegmentGranularity();
       }
-      granularitySpec = new ClientCompactionTaskGranularitySpec(
+      final ClientCompactionTaskGranularitySpec granularitySpec = new ClientCompactionTaskGranularitySpec(
           segmentGranularityToUse,
           config.getGranularitySpec() != null ? config.getGranularitySpec().getQueryGranularity() : null,
           config.getGranularitySpec() != null ? config.getGranularitySpec().isRollup() : null
       );
 
       // Create dimensionsSpec to send to compaction task
-      ClientCompactionTaskDimensionsSpec dimensionsSpec;
+      final ClientCompactionTaskDimensionsSpec dimensionsSpec;
       if (config.getDimensionsSpec() != null) {
         dimensionsSpec = new ClientCompactionTaskDimensionsSpec(
             config.getDimensionsSpec().getDimensions()
@@ -506,7 +548,7 @@ public class CompactSegments implements CoordinatorCustomDuty
       }
 
       final String taskId = compactSegments(
-          segmentsToCompact,
+          entry,
           config.getTaskPriority(),
           ClientCompactionTaskQueryTuningConfig.from(
               config.getTuningConfig(),
@@ -552,28 +594,25 @@ public class CompactSegments implements CoordinatorCustomDuty
   {
     // Mark all the segments remaining in the iterator as "awaiting compaction"
     while (iterator.hasNext()) {
-      final SegmentsToCompact entry = iterator.next();
-      if (!entry.isEmpty()) {
-        final String dataSourceName = entry.getFirst().getDataSource();
-        currentRunAutoCompactionSnapshotBuilders
-            .computeIfAbsent(dataSourceName, AutoCompactionSnapshot::builder)
-            .incrementWaitingStats(entry.getStats());
-      }
+      final CompactionCandidate entry = iterator.next();
+      currentRunAutoCompactionSnapshotBuilders
+          .computeIfAbsent(entry.getDataSource(), AutoCompactionSnapshot::builder)
+          .incrementWaitingStats(entry.getStats());
     }
 
     // Statistics of all segments considered compacted after this run
-    iterator.totalCompactedStatistics().forEach((dataSource, compactedStats) -> {
-      currentRunAutoCompactionSnapshotBuilders
-          .computeIfAbsent(dataSource, AutoCompactionSnapshot::builder)
-          .incrementCompactedStats(compactedStats);
-    });
+    iterator.getCompactedSegments().forEach(
+        candidateSegments -> currentRunAutoCompactionSnapshotBuilders
+            .computeIfAbsent(candidateSegments.getDataSource(), AutoCompactionSnapshot::builder)
+            .incrementCompactedStats(candidateSegments.getStats())
+    );
 
     // Statistics of all segments considered skipped after this run
-    iterator.totalSkippedStatistics().forEach((dataSource, dataSourceSkippedStatistics) -> {
-      currentRunAutoCompactionSnapshotBuilders
-          .computeIfAbsent(dataSource, AutoCompactionSnapshot::builder)
-          .incrementSkippedStats(dataSourceSkippedStatistics);
-    });
+    iterator.getSkippedSegments().forEach(
+        candidateSegments -> currentRunAutoCompactionSnapshotBuilders
+            .computeIfAbsent(candidateSegments.getDataSource(), AutoCompactionSnapshot::builder)
+            .incrementSkippedStats(candidateSegments.getStats())
+    );
 
     final Map<String, AutoCompactionSnapshot> currentAutoCompactionSnapshotPerDataSource = new HashMap<>();
     currentRunAutoCompactionSnapshotBuilders.forEach((dataSource, builder) -> {
@@ -604,16 +643,6 @@ public class CompactSegments implements CoordinatorCustomDuty
     stats.add(Stats.Compaction.SKIPPED_INTERVALS, rowKey, autoCompactionSnapshot.getIntervalCountSkipped());
   }
 
-  @Nullable
-  public Long getTotalSizeOfSegmentsAwaitingCompaction(String dataSource)
-  {
-    AutoCompactionSnapshot autoCompactionSnapshot = autoCompactionSnapshotPerDataSource.get().get(dataSource);
-    if (autoCompactionSnapshot == null) {
-      return null;
-    }
-    return autoCompactionSnapshot.getBytesAwaitingCompaction();
-  }
-
   @Nullable
   public AutoCompactionSnapshot getAutoCompactionSnapshot(String dataSource)
   {
@@ -626,10 +655,10 @@ public class CompactSegments implements CoordinatorCustomDuty
   }
 
   private String compactSegments(
-      List<DataSegment> segments,
+      CompactionCandidate entry,
       int compactionTaskPriority,
-      @Nullable ClientCompactionTaskQueryTuningConfig tuningConfig,
-      @Nullable ClientCompactionTaskGranularitySpec granularitySpec,
+      ClientCompactionTaskQueryTuningConfig tuningConfig,
+      ClientCompactionTaskGranularitySpec granularitySpec,
       @Nullable ClientCompactionTaskDimensionsSpec dimensionsSpec,
       @Nullable AggregatorFactory[] metricsSpec,
       @Nullable ClientCompactionTaskTransformSpec transformSpec,
@@ -638,6 +667,7 @@ public class CompactSegments implements CoordinatorCustomDuty
       ClientCompactionRunnerInfo compactionRunner
   )
   {
+    final List<DataSegment> segments = entry.getSegments();
     Preconditions.checkArgument(!segments.isEmpty(), "Expect non-empty segments to compact");
 
     final String dataSource = segments.get(0).getDataSource();
@@ -651,7 +681,7 @@ public class CompactSegments implements CoordinatorCustomDuty
 
     final String taskId = IdUtils.newTaskId(TASK_ID_PREFIX, ClientCompactionTaskQuery.TYPE, dataSource, null);
     final Granularity segmentGranularity = granularitySpec == null ? null : granularitySpec.getSegmentGranularity();
-    final ClientTaskQuery taskPayload = new ClientCompactionTaskQuery(
+    final ClientCompactionTaskQuery taskPayload = new ClientCompactionTaskQuery(
         taskId,
         dataSource,
         new ClientCompactionIOConfig(
@@ -667,6 +697,8 @@ public class CompactSegments implements CoordinatorCustomDuty
         compactionRunner
     );
     FutureUtils.getUnchecked(overlordClient.runTask(taskId, taskPayload), true);
+    statusTracker.onTaskSubmitted(taskPayload, entry);
+
     return taskId;
   }
 }
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CoordinatorDutyUtils.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CoordinatorDutyUtils.java
index f6f31173fa5..1c31a28087b 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CoordinatorDutyUtils.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CoordinatorDutyUtils.java
@@ -86,17 +86,16 @@ public class CoordinatorDutyUtils
   }
 
   /**
-   * Return the number of active tasks that match the task predicate provided. The number of active tasks returned
-   * may be an overestimate, as tasks that return status's with null types will be conservatively counted to match the
-   * predicate provided.
+   * Fetches active task statuses that match the given predicate.
+   * Task statuses with null types are considered to satisfy the predicate too.
    *
    * @param overlordClient The overlord client to use to retrieve the list of active tasks.
-   * @param taskPredicate  The predicate to match against the list of retreived task status.
+   * @param taskPredicate  The predicate to match against the list of retrieved task statuses.
    *                       This predicate will never be called with a null task status.
    *
-   * @return the number of active tasks that match the task predicate provided
+   * @return Active task statuses that match the given predicate.
    */
-  public static List<TaskStatusPlus> getNumActiveTaskSlots(
+  public static List<TaskStatusPlus> getStatusOfActiveTasks(
       @Nonnull final OverlordClient overlordClient,
       final Predicate<TaskStatusPlus> taskPredicate
   )
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java
index 64b61df5e53..199c3555b37 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java
@@ -306,7 +306,7 @@ public class KillUnusedSegments implements CoordinatorDuty
 
     final int availableKillTaskSlots = Math.max(
         0,
-        killTaskCapacity - CoordinatorDutyUtils.getNumActiveTaskSlots(overlordClient, IS_AUTO_KILL_TASK).size()
+        killTaskCapacity - CoordinatorDutyUtils.getStatusOfActiveTasks(overlordClient, IS_AUTO_KILL_TASK).size()
     );
 
     stats.add(Stats.Kill.AVAILABLE_SLOTS, availableKillTaskSlots);
diff --git a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java
index 7bd4ee85124..8f9dfb9ca85 100644
--- a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java
+++ b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java
@@ -146,6 +146,7 @@ public class CoordinatorCompactionConfigsResource
             compactionTaskSlotRatio,
             maxCompactionTaskSlots,
             useAutoScaleSlots,
+            null,
             null
         ),
         req
diff --git a/server/src/main/java/org/apache/druid/server/http/CompactionResource.java b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionResource.java
similarity index 53%
rename from server/src/main/java/org/apache/druid/server/http/CompactionResource.java
rename to server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionResource.java
index e88d0cdacfb..015a27435cf 100644
--- a/server/src/main/java/org/apache/druid/server/http/CompactionResource.java
+++ b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionResource.java
@@ -22,13 +22,19 @@ package org.apache.druid.server.http;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.ListenableFuture;
 import com.google.inject.Inject;
 import com.sun.jersey.spi.container.ResourceFilters;
+import org.apache.druid.common.guava.FutureUtils;
+import org.apache.druid.rpc.HttpResponseException;
+import org.apache.druid.rpc.indexing.OverlordClient;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.coordinator.ClusterCompactionConfig;
 import org.apache.druid.server.coordinator.DruidCoordinator;
 import org.apache.druid.server.http.security.ConfigResourceFilter;
 import org.apache.druid.server.http.security.StateResourceFilter;
 
+import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
@@ -39,16 +45,19 @@ import javax.ws.rs.core.Response;
 import java.util.Collection;
 
 @Path("/druid/coordinator/v1/compaction")
-public class CompactionResource
+public class CoordinatorCompactionResource
 {
   private final DruidCoordinator coordinator;
+  private final OverlordClient overlordClient;
 
   @Inject
-  public CompactionResource(
-      DruidCoordinator coordinator
+  public CoordinatorCompactionResource(
+      DruidCoordinator coordinator,
+      OverlordClient overlordClient
   )
   {
     this.coordinator = coordinator;
+    this.overlordClient = overlordClient;
   }
 
   /**
@@ -72,11 +81,21 @@ public class CompactionResource
       @QueryParam("dataSource") String dataSource
   )
   {
-    final Long notCompactedSegmentSizeBytes = coordinator.getTotalSizeOfSegmentsAwaitingCompaction(dataSource);
-    if (notCompactedSegmentSizeBytes == null) {
-      return Response.status(Response.Status.NOT_FOUND).entity(ImmutableMap.of("error", "unknown dataSource")).build();
+    if (dataSource == null || dataSource.isEmpty()) {
+      return Response.status(Response.Status.BAD_REQUEST)
+                     .entity(ImmutableMap.of("error", "No DataSource specified"))
+                     .build();
+    }
+
+    if (isCompactionSupervisorEnabled()) {
+      return buildResponse(overlordClient.getBytesAwaitingCompaction(dataSource));
+    }
+
+    final AutoCompactionSnapshot snapshot = coordinator.getAutoCompactionSnapshotForDataSource(dataSource);
+    if (snapshot == null) {
+      return Response.status(Response.Status.NOT_FOUND).entity(ImmutableMap.of("error", "Unknown DataSource")).build();
     } else {
-      return Response.ok(ImmutableMap.of("remainingSegmentSize", notCompactedSegmentSizeBytes)).build();
+      return Response.ok(ImmutableMap.of("remainingSegmentSize", snapshot.getBytesAwaitingCompaction())).build();
     }
   }
 
@@ -88,16 +107,66 @@ public class CompactionResource
       @QueryParam("dataSource") String dataSource
   )
   {
+    if (isCompactionSupervisorEnabled()) {
+      return buildResponse(overlordClient.getCompactionSnapshots(dataSource));
+    }
+
     final Collection<AutoCompactionSnapshot> snapshots;
     if (dataSource == null || dataSource.isEmpty()) {
       snapshots = coordinator.getAutoCompactionSnapshot().values();
     } else {
       AutoCompactionSnapshot autoCompactionSnapshot = coordinator.getAutoCompactionSnapshotForDataSource(dataSource);
       if (autoCompactionSnapshot == null) {
-        return Response.status(Response.Status.NOT_FOUND).entity(ImmutableMap.of("error", "unknown dataSource")).build();
+        return Response.status(Response.Status.NOT_FOUND).entity(ImmutableMap.of("error", "Unknown DataSource")).build();
       }
       snapshots = ImmutableList.of(autoCompactionSnapshot);
     }
     return Response.ok(ImmutableMap.of("latestStatus", snapshots)).build();
   }
+
+  @POST
+  @Path("/simulate")
+  @Consumes(MediaType.APPLICATION_JSON)
+  @ResourceFilters(StateResourceFilter.class)
+  public Response simulateWithClusterConfigUpdate(
+      ClusterCompactionConfig updatePayload
+  )
+  {
+    return Response.ok().entity(
+        coordinator.simulateRunWithConfigUpdate(updatePayload)
+    ).build();
+  }
+
+  private <T> Response buildResponse(ListenableFuture<T> future)
+  {
+    try {
+      return Response.ok(FutureUtils.getUnchecked(future, true)).build();
+    }
+    catch (Exception e) {
+      if (e.getCause() instanceof HttpResponseException) {
+        final HttpResponseException cause = (HttpResponseException) e.getCause();
+        return Response.status(cause.getResponse().getStatus().getCode())
+                       .entity(cause.getResponse().getContent())
+                       .build();
+      } else {
+        return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
+                       .entity(ImmutableMap.of("error", e.getMessage()))
+                       .build();
+      }
+    }
+  }
+
+  /**
+   * Check if compaction supervisors are enabled on the Overlord. 
+   */
+  private boolean isCompactionSupervisorEnabled()
+  {
+    try {
+      return FutureUtils.getUnchecked(overlordClient.isCompactionSupervisorEnabled(), true);
+    }
+    catch (Exception e) {
+      // Overlord is probably on an older version, assume that compaction supervisor is not enabled
+      return false;
+    }
+  }
 }
diff --git a/server/src/test/java/org/apache/druid/client/indexing/NoopOverlordClient.java b/server/src/test/java/org/apache/druid/client/indexing/NoopOverlordClient.java
index 10ebeb53af2..c73ff1ca059 100644
--- a/server/src/test/java/org/apache/druid/client/indexing/NoopOverlordClient.java
+++ b/server/src/test/java/org/apache/druid/client/indexing/NoopOverlordClient.java
@@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.rpc.ServiceRetryPolicy;
 import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -122,6 +123,24 @@ public class NoopOverlordClient implements OverlordClient
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource)
+  {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListenableFuture<Boolean> isCompactionSupervisorEnabled()
+  {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public OverlordClient withRetryPolicy(ServiceRetryPolicy retryPolicy)
   {
diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
index 6eccbccaa84..f82cfbf2a04 100644
--- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
@@ -35,6 +35,7 @@ import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.segment.SegmentSchemaMapping;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
 import org.apache.druid.segment.metadata.FingerprintGenerator;
 import org.apache.druid.segment.metadata.SegmentSchemaManager;
@@ -174,7 +175,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
       pendingSegmentsForTask.add(
           new PendingSegmentRecord(
               new SegmentIdWithShardSpec(
-                  DS.WIKI,
+                  TestDataSource.WIKI,
                   Intervals.of("2023-01-01/2023-02-01"),
                   alreadyUpgradedVersion,
                   new NumberedShardSpec(i, 0)
@@ -209,7 +210,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
       pendingSegmentsForTask.add(
           new PendingSegmentRecord(
               new SegmentIdWithShardSpec(
-                  DS.WIKI,
+                  TestDataSource.WIKI,
                   Intervals.of("2023-01-01/2023-02-01"),
                   alreadyUpgradedVersion,
                   new NumberedShardSpec(10 + i, 0)
@@ -243,7 +244,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
       pendingSegmentsForTask.add(
           new PendingSegmentRecord(
               new SegmentIdWithShardSpec(
-                  DS.WIKI,
+                  TestDataSource.WIKI,
                   Intervals.of("2023-01-01/2023-02-01"),
                   alreadyUpgradedVersion,
                   new NumberedShardSpec(20 + i, 0)
@@ -257,7 +258,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     }
 
     derbyConnector.retryWithHandle(
-        handle -> coordinator.insertPendingSegmentsIntoMetastore(handle, pendingSegmentsForTask, DS.WIKI, false)
+        handle -> coordinator.insertPendingSegmentsIntoMetastore(handle, pendingSegmentsForTask, TestDataSource.WIKI, false)
     );
 
     final Map<DataSegment, ReplaceTaskLock> segmentToReplaceLock
@@ -272,7 +273,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     Set<DataSegment> allCommittedSegments
         = new HashSet<>(retrieveUsedSegments(derbyConnectorRule.metadataTablesConfigSupplier().get()));
     Map<String, String> upgradedFromSegmentIdMap = coordinator.retrieveUpgradedFromSegmentIds(
-        DS.WIKI,
+        TestDataSource.WIKI,
         allCommittedSegments.stream().map(DataSegment::getId).map(SegmentId::toString).collect(Collectors.toSet())
     );
     // Verify the segments present in the metadata store
@@ -901,7 +902,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     final List<Interval> intervals = segments.stream().map(DataSegment::getInterval).collect(Collectors.toList());
 
     final Collection<DataSegment> actualUsedSegments = coordinator.retrieveUsedSegmentsForIntervals(
-        DS.WIKI,
+        TestDataSource.WIKI,
         intervals,
         Segments.ONLY_VISIBLE
     );
@@ -920,7 +921,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
                               .anyMatch(segment -> !segment.getInterval().overlaps(outOfRangeInterval)));
 
     final Collection<DataSegment> actualUsedSegments = coordinator.retrieveUsedSegmentsForIntervals(
-        DS.WIKI,
+        TestDataSource.WIKI,
         ImmutableList.of(outOfRangeInterval),
         Segments.ONLY_VISIBLE
     );
@@ -934,7 +935,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     final List<DataSegment> segments = createAndGetUsedYearSegments(1900, 2133);
 
     final Collection<DataSegment> actualUsedSegments = coordinator.retrieveAllUsedSegments(
-        DS.WIKI,
+        TestDataSource.WIKI,
         Segments.ONLY_VISIBLE
     );
 
@@ -949,7 +950,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     markAllSegmentsUnused(new HashSet<>(segments), DateTimes.nowUtc());
 
     final List<DataSegment> actualUnusedSegments = coordinator.retrieveUnusedSegmentsForInterval(
-        DS.WIKI,
+        TestDataSource.WIKI,
         Intervals.of("1900/3000"),
         null,
         null
@@ -967,7 +968,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
     final int requestedLimit = segments.size();
     final List<DataSegment> actualUnusedSegments = coordinator.retrieveUnusedSegmentsForInterval(
-        DS.WIKI,
+        TestDataSource.WIKI,
         Intervals.of("1900/3000"),
         requestedLimit,
         null
@@ -985,7 +986,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
     final int requestedLimit = segments.size() - 1;
     final List<DataSegment> actualUnusedSegments = coordinator.retrieveUnusedSegmentsForInterval(
-        DS.WIKI,
+        TestDataSource.WIKI,
         Intervals.of("1900/3000"),
         requestedLimit,
         null
@@ -1003,7 +1004,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
     final int limit = segments.size() + 1;
     final List<DataSegment> actualUnusedSegments = coordinator.retrieveUnusedSegmentsForInterval(
-        DS.WIKI,
+        TestDataSource.WIKI,
         Intervals.of("1900/3000"),
         limit,
         null
@@ -1024,7 +1025,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     final int limit = segments.size() + 1;
 
     final List<DataSegment> actualUnusedSegments = coordinator.retrieveUnusedSegmentsForInterval(
-        DS.WIKI,
+        TestDataSource.WIKI,
         outOfRangeInterval,
         limit,
         null
@@ -1540,7 +1541,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     for (DataSegment unusedSegment : unusedSegments) {
       Assertions.assertThat(
           coordinator.retrieveUnusedSegmentsForInterval(
-              DS.WIKI,
+              TestDataSource.WIKI,
               Intervals.of("2023-01-01/2023-01-04"),
               ImmutableList.of(unusedSegment.getVersion()),
               null,
@@ -1551,7 +1552,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
     Assertions.assertThat(
         coordinator.retrieveUnusedSegmentsForInterval(
-            DS.WIKI,
+            TestDataSource.WIKI,
             Intervals.of("2023-01-01/2023-01-04"),
             ImmutableList.of(v1, v2),
             null,
@@ -1561,7 +1562,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
     Assertions.assertThat(
         coordinator.retrieveUnusedSegmentsForInterval(
-            DS.WIKI,
+            TestDataSource.WIKI,
             Intervals.of("2023-01-01/2023-01-04"),
             null,
             null,
@@ -1571,7 +1572,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
     Assertions.assertThat(
         coordinator.retrieveUnusedSegmentsForInterval(
-            DS.WIKI,
+            TestDataSource.WIKI,
             Intervals.of("2023-01-01/2023-01-04"),
             ImmutableList.of("some-non-existent-version"),
               null,
@@ -3235,7 +3236,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
     // Allocate and commit a data segment by appending to the same interval
     final SegmentIdWithShardSpec identifier = coordinator.allocatePendingSegment(
-        DS.WIKI,
+        TestDataSource.WIKI,
         "seq",
         tombstoneSegment.getVersion(),
         interval,
@@ -3260,7 +3261,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     markAllSegmentsUnused(tombstones, DateTimes.nowUtc());
 
     final Collection<DataSegment> allUsedSegments = coordinator.retrieveAllUsedSegments(
-        DS.WIKI,
+        TestDataSource.WIKI,
         Segments.ONLY_VISIBLE
     );
 
@@ -3290,7 +3291,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
     // Allocate and commit a data segment by appending to the same interval
     final SegmentIdWithShardSpec identifier = coordinator.allocatePendingSegment(
-        DS.WIKI,
+        TestDataSource.WIKI,
         "seq",
         tombstoneSegment.getVersion(),
         interval,
@@ -3315,7 +3316,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     markAllSegmentsUnused(tombstones, DateTimes.nowUtc());
 
     final Collection<DataSegment> allUsedSegments = coordinator.retrieveAllUsedSegments(
-        DS.WIKI,
+        TestDataSource.WIKI,
         Segments.ONLY_VISIBLE
     );
 
@@ -3329,7 +3330,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
   public void testSegmentIdShouldNotBeReallocated()
   {
     final SegmentIdWithShardSpec idWithNullTaskAllocator = coordinator.allocatePendingSegment(
-        DS.WIKI,
+        TestDataSource.WIKI,
         "seq",
         "0",
         Intervals.ETERNITY,
@@ -3345,7 +3346,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     );
 
     final SegmentIdWithShardSpec idWithValidTaskAllocator = coordinator.allocatePendingSegment(
-        DS.WIKI,
+        TestDataSource.WIKI,
         "seq",
         "1",
         Intervals.ETERNITY,
@@ -3363,12 +3364,12 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
     // Insert pending segments
     coordinator.commitSegments(ImmutableSet.of(dataSegment0, dataSegment1), null);
     // Clean up pending segments corresponding to the valid task allocator id
-    coordinator.deletePendingSegmentsForTaskAllocatorId(DS.WIKI, "taskAllocatorId");
+    coordinator.deletePendingSegmentsForTaskAllocatorId(TestDataSource.WIKI, "taskAllocatorId");
     // Mark all segments as unused
-    coordinator.markSegmentsAsUnusedWithinInterval(DS.WIKI, Intervals.ETERNITY);
+    coordinator.markSegmentsAsUnusedWithinInterval(TestDataSource.WIKI, Intervals.ETERNITY);
 
     final SegmentIdWithShardSpec theId = coordinator.allocatePendingSegment(
-        DS.WIKI,
+        TestDataSource.WIKI,
         "seq",
         "2",
         Intervals.ETERNITY,
@@ -3406,7 +3407,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
         ),
         null
     );
-    coordinator.markSegmentsAsUnusedWithinInterval(DS.WIKI, Intervals.ETERNITY);
+    coordinator.markSegmentsAsUnusedWithinInterval(TestDataSource.WIKI, Intervals.ETERNITY);
 
     DataSegment usedSegmentForExactIntervalAndVersion = createSegment(
         Intervals.of("2024/2025"),
@@ -3417,7 +3418,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
 
 
     List<String> unusedSegmentIdsForIntervalAndVersion =
-        coordinator.retrieveUnusedSegmentIdsForExactIntervalAndVersion(DS.WIKI, Intervals.of("2024/2025"), "v1");
+        coordinator.retrieveUnusedSegmentIdsForExactIntervalAndVersion(TestDataSource.WIKI, Intervals.of("2024/2025"), "v1");
     Assert.assertEquals(1, unusedSegmentIdsForIntervalAndVersion.size());
     Assert.assertEquals(
         unusedSegmentForExactIntervalAndVersion.getId().toString(),
diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java
index 2076e5ffa46..cb0bea36c20 100644
--- a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java
+++ b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java
@@ -32,6 +32,7 @@ import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.jackson.JacksonUtils;
 import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.segment.SegmentSchemaMapping;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
 import org.apache.druid.segment.metadata.FingerprintGenerator;
@@ -310,15 +311,10 @@ public class IndexerSqlMetadataStorageCoordinatorTestBase
   protected FingerprintGenerator fingerprintGenerator;
   protected SegmentSchemaTestUtils segmentSchemaTestUtils;
 
-  protected static class DS
-  {
-    static final String WIKI = "wiki";
-  }
-
   protected DataSegment createSegment(Interval interval, String version, ShardSpec shardSpec)
   {
     return DataSegment.builder()
-                      .dataSource(DS.WIKI)
+                      .dataSource(TestDataSource.WIKI)
                       .interval(interval)
                       .version(version)
                       .shardSpec(shardSpec)
@@ -365,7 +361,7 @@ public class IndexerSqlMetadataStorageCoordinatorTestBase
                                                tablesConfig,
                                                mapper
                                            )
-                                           .retrieveUnusedSegments(DS.WIKI, intervals, null, limit, lastSegmentId, sortOrder, maxUsedStatusLastUpdatedTime)) {
+                                           .retrieveUnusedSegments(TestDataSource.WIKI, intervals, null, limit, lastSegmentId, sortOrder, maxUsedStatusLastUpdatedTime)) {
             return ImmutableList.copyOf(iterator);
           }
         }
@@ -384,13 +380,8 @@ public class IndexerSqlMetadataStorageCoordinatorTestBase
     return derbyConnector.inReadOnlyTransaction(
         (handle, status) -> {
           try (final CloseableIterator<DataSegmentPlus> iterator =
-                   SqlSegmentsMetadataQuery.forHandle(
-                                               handle,
-                                               derbyConnector,
-                                               tablesConfig,
-                                               mapper
-                                           )
-                                           .retrieveUnusedSegmentsPlus(DS.WIKI, intervals, null, limit, lastSegmentId, sortOrder, maxUsedStatusLastUpdatedTime)) {
+                   SqlSegmentsMetadataQuery.forHandle(handle, derbyConnector, tablesConfig, mapper)
+                                           .retrieveUnusedSegmentsPlus(TestDataSource.WIKI, intervals, null, limit, lastSegmentId, sortOrder, maxUsedStatusLastUpdatedTime)) {
             return ImmutableList.copyOf(iterator);
           }
         }
diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java
index efeadbdc04e..82a87dd15a9 100644
--- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java
@@ -33,6 +33,7 @@ import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.audit.AuditSerdeHelper;
 import org.apache.druid.server.audit.SQLAuditManager;
 import org.apache.druid.server.audit.SQLAuditManagerConfig;
@@ -52,8 +53,6 @@ import java.util.Map;
 
 public class SQLMetadataRuleManagerTest
 {
-  private static final String DATASOURCE = "wiki";
-  
   @org.junit.Rule
   public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new TestDerbyConnector.DerbyConnectorRule();
 
@@ -108,12 +107,12 @@ public class SQLMetadataRuleManagerTest
             null
         )
     );
-    ruleManager.overrideRule(DATASOURCE, rules, createAuditInfo("override rule"));
+    ruleManager.overrideRule(TestDataSource.WIKI, rules, createAuditInfo("override rule"));
     // New rule should be be reflected in the in memory rules map immediately after being set by user
     Map<String, List<Rule>> allRules = ruleManager.getAllRules();
     Assert.assertEquals(1, allRules.size());
-    Assert.assertEquals(1, allRules.get(DATASOURCE).size());
-    Assert.assertEquals(rules.get(0), allRules.get(DATASOURCE).get(0));
+    Assert.assertEquals(1, allRules.get(TestDataSource.WIKI).size());
+    Assert.assertEquals(rules.get(0), allRules.get(TestDataSource.WIKI).get(0));
   }
 
   @Test
@@ -122,7 +121,7 @@ public class SQLMetadataRuleManagerTest
     // Datasource level rules cannot be null
     IAE exception = Assert.assertThrows(
         IAE.class,
-        () -> ruleManager.overrideRule(DATASOURCE, null, createAuditInfo("null rule"))
+        () -> ruleManager.overrideRule(TestDataSource.WIKI, null, createAuditInfo("null rule"))
     );
     Assert.assertEquals("Rules cannot be null.", exception.getMessage());
 
@@ -155,7 +154,7 @@ public class SQLMetadataRuleManagerTest
     // Datasource level rules can be empty
     Assert.assertTrue(
         ruleManager.overrideRule(
-            DATASOURCE,
+            TestDataSource.WIKI,
             Collections.emptyList(),
             createAuditInfo("empty rule")
         )
@@ -173,14 +172,14 @@ public class SQLMetadataRuleManagerTest
         )
     );
     final AuditInfo auditInfo = createAuditInfo("create audit entry");
-    ruleManager.overrideRule(DATASOURCE, rules, auditInfo);
+    ruleManager.overrideRule(TestDataSource.WIKI, rules, auditInfo);
     // fetch rules from metadata storage
     ruleManager.poll();
 
-    Assert.assertEquals(rules, ruleManager.getRules(DATASOURCE));
+    Assert.assertEquals(rules, ruleManager.getRules(TestDataSource.WIKI));
 
     // verify audit entry is created
-    List<AuditEntry> auditEntries = auditManager.fetchAuditHistory(DATASOURCE, "rules", null);
+    List<AuditEntry> auditEntries = auditManager.fetchAuditHistory(TestDataSource.WIKI, "rules", null);
     Assert.assertEquals(1, auditEntries.size());
     AuditEntry entry = auditEntries.get(0);
 
@@ -189,7 +188,7 @@ public class SQLMetadataRuleManagerTest
         mapper.readValue(entry.getPayload().serialized(), new TypeReference<List<Rule>>() {})
     );
     Assert.assertEquals(auditInfo, entry.getAuditInfo());
-    Assert.assertEquals(DATASOURCE, entry.getKey());
+    Assert.assertEquals(TestDataSource.WIKI, entry.getKey());
   }
 
   @Test
@@ -205,12 +204,12 @@ public class SQLMetadataRuleManagerTest
         )
     );
     final AuditInfo auditInfo = createAuditInfo("test_comment");
-    ruleManager.overrideRule(DATASOURCE, rules, auditInfo);
+    ruleManager.overrideRule(TestDataSource.WIKI, rules, auditInfo);
     ruleManager.overrideRule("test_dataSource2", rules, auditInfo);
     // fetch rules from metadata storage
     ruleManager.poll();
 
-    Assert.assertEquals(rules, ruleManager.getRules(DATASOURCE));
+    Assert.assertEquals(rules, ruleManager.getRules(TestDataSource.WIKI));
     Assert.assertEquals(rules, ruleManager.getRules("test_dataSource2"));
 
     // test fetch audit entries
@@ -235,13 +234,13 @@ public class SQLMetadataRuleManagerTest
             null
         )
     );
-    ruleManager.overrideRule(DATASOURCE, rules, createAuditInfo("test"));
+    ruleManager.overrideRule(TestDataSource.WIKI, rules, createAuditInfo("test"));
 
     // Verify that the rule was added
     ruleManager.poll();
     Map<String, List<Rule>> allRules = ruleManager.getAllRules();
     Assert.assertEquals(1, allRules.size());
-    Assert.assertEquals(1, allRules.get(DATASOURCE).size());
+    Assert.assertEquals(1, allRules.get(TestDataSource.WIKI).size());
 
     // Now delete rules
     ruleManager.removeRulesForEmptyDatasourcesOlderThan(System.currentTimeMillis());
@@ -262,13 +261,13 @@ public class SQLMetadataRuleManagerTest
             null
         )
     );
-    ruleManager.overrideRule(DATASOURCE, rules, createAuditInfo("update rules"));
+    ruleManager.overrideRule(TestDataSource.WIKI, rules, createAuditInfo("update rules"));
 
     // Verify that rule was added
     ruleManager.poll();
     Map<String, List<Rule>> allRules = ruleManager.getAllRules();
     Assert.assertEquals(1, allRules.size());
-    Assert.assertEquals(1, allRules.get(DATASOURCE).size());
+    Assert.assertEquals(1, allRules.get(TestDataSource.WIKI).size());
 
     // This will not delete the rule as the rule was created just now so it will have the created timestamp later than
     // the timestamp 2012-01-01T00:00:00Z
@@ -278,7 +277,7 @@ public class SQLMetadataRuleManagerTest
     ruleManager.poll();
     allRules = ruleManager.getAllRules();
     Assert.assertEquals(1, allRules.size());
-    Assert.assertEquals(1, allRules.get(DATASOURCE).size());
+    Assert.assertEquals(1, allRules.get(TestDataSource.WIKI).size());
   }
 
   @Test
@@ -291,17 +290,17 @@ public class SQLMetadataRuleManagerTest
             null
         )
     );
-    ruleManager.overrideRule(DATASOURCE, rules, createAuditInfo("update rules"));
+    ruleManager.overrideRule(TestDataSource.WIKI, rules, createAuditInfo("update rules"));
 
     // Verify that rule was added
     ruleManager.poll();
     Map<String, List<Rule>> allRules = ruleManager.getAllRules();
     Assert.assertEquals(1, allRules.size());
-    Assert.assertEquals(1, allRules.get(DATASOURCE).size());
+    Assert.assertEquals(1, allRules.get(TestDataSource.WIKI).size());
 
     // Add segment metadata to segment table so that the datasource is considered active
     DataSegment dataSegment = new DataSegment(
-        DATASOURCE,
+        TestDataSource.WIKI,
         Intervals.of("2015-01-01/2015-02-01"),
         "1",
         ImmutableMap.of(
@@ -324,7 +323,7 @@ public class SQLMetadataRuleManagerTest
     ruleManager.poll();
     allRules = ruleManager.getAllRules();
     Assert.assertEquals(1, allRules.size());
-    Assert.assertEquals(1, allRules.get(DATASOURCE).size());
+    Assert.assertEquals(1, allRules.get(TestDataSource.WIKI).size());
   }
 
   @Test
diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java
index d101a7a74b4..16a0747acaa 100644
--- a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java
@@ -33,6 +33,7 @@ import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.emitter.EmittingLogger;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
 import org.apache.druid.segment.metadata.SegmentSchemaCache;
 import org.apache.druid.segment.metadata.SegmentSchemaManager;
@@ -58,12 +59,6 @@ import java.util.stream.Collectors;
 
 public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTestBase
 {
-  private static class DS
-  {
-    static final String WIKI = "wikipedia";
-    static final String KOALA = "koala";
-  }
-
   private static DataSegment createSegment(
       String dataSource,
       String interval,
@@ -88,9 +83,9 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
       = new TestDerbyConnector.DerbyConnectorRule();
 
   private final DataSegment wikiSegment1 =
-      CreateDataSegments.ofDatasource(DS.WIKI).startingAt("2012-03-15").eachOfSizeInMb(500).get(0);
+      CreateDataSegments.ofDatasource(TestDataSource.WIKI).startingAt("2012-03-15").eachOfSizeInMb(500).get(0);
   private final DataSegment wikiSegment2 =
-      CreateDataSegments.ofDatasource(DS.WIKI).startingAt("2012-01-05").eachOfSizeInMb(500).get(0);
+      CreateDataSegments.ofDatasource(TestDataSource.WIKI).startingAt("2012-01-05").eachOfSizeInMb(500).get(0);
 
   private void publishUnusedSegments(DataSegment... segments) throws IOException
   {
@@ -165,7 +160,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
             .stream()
             .map(ImmutableDruidDataSource::getName).count()
     );
-    Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(DS.WIKI));
+    Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.WIKI));
     Assert.assertTrue(
         ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments()).isEmpty()
     );
@@ -184,11 +179,11 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.PeriodicDatabasePoll);
     dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
     Assert.assertEquals(
-        ImmutableSet.of(DS.WIKI),
+        ImmutableSet.of(TestDataSource.WIKI),
         sqlSegmentsMetadataManager.retrieveAllDataSourceNames()
     );
     Assert.assertEquals(
-        ImmutableList.of(DS.WIKI),
+        ImmutableList.of(TestDataSource.WIKI),
         dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
                            .stream()
                            .map(ImmutableDruidDataSource::getName)
@@ -196,7 +191,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     );
     Assert.assertEquals(
         ImmutableSet.of(wikiSegment1, wikiSegment2),
-        ImmutableSet.copyOf(dataSourcesSnapshot.getDataSource(DS.WIKI).getSegments())
+        ImmutableSet.copyOf(dataSourcesSnapshot.getDataSource(TestDataSource.WIKI).getSegments())
     );
     Assert.assertEquals(
         ImmutableSet.of(wikiSegment1, wikiSegment2),
@@ -219,11 +214,11 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.OnDemandDatabasePoll);
     dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
     Assert.assertEquals(
-        ImmutableSet.of(DS.WIKI),
+        ImmutableSet.of(TestDataSource.WIKI),
         sqlSegmentsMetadataManager.retrieveAllDataSourceNames()
     );
     Assert.assertEquals(
-        ImmutableList.of(DS.WIKI),
+        ImmutableList.of(TestDataSource.WIKI),
         dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
                            .stream()
                            .map(ImmutableDruidDataSource::getName)
@@ -231,7 +226,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     );
     Assert.assertEquals(
         ImmutableSet.of(wikiSegment1, wikiSegment2),
-        ImmutableSet.copyOf(dataSourcesSnapshot.getDataSource(DS.WIKI).getSegments())
+        ImmutableSet.copyOf(dataSourcesSnapshot.getDataSource(TestDataSource.WIKI).getSegments())
     );
     Assert.assertEquals(
         ImmutableSet.of(wikiSegment1, wikiSegment2),
@@ -252,13 +247,13 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.PeriodicDatabasePoll);
     dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
     Assert.assertEquals(
-        ImmutableList.of(DS.WIKI),
+        ImmutableList.of(TestDataSource.WIKI),
         dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
                            .stream()
                            .map(ImmutableDruidDataSource::getName)
                            .collect(Collectors.toList())
     );
-    publishSegment(createNewSegment1(DS.KOALA));
+    publishSegment(createNewSegment1(TestDataSource.KOALA));
 
     // This call will force on demand poll
     sqlSegmentsMetadataManager.forceOrWaitOngoingDatabasePoll();
@@ -267,7 +262,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     // New datasource should now be in the snapshot since we just force on demand poll.
     dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
     Assert.assertEquals(
-        ImmutableList.of(DS.KOALA, DS.WIKI),
+        ImmutableList.of(TestDataSource.KOALA, TestDataSource.WIKI),
         dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
                            .stream()
                            .map(ImmutableDruidDataSource::getName)
@@ -285,7 +280,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.PeriodicDatabasePoll);
     dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
     Assert.assertEquals(
-        ImmutableSet.of(DS.KOALA, "wikipedia3", DS.WIKI),
+        ImmutableSet.of(TestDataSource.KOALA, "wikipedia3", TestDataSource.WIKI),
         dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
                            .stream()
                            .map(ImmutableDruidDataSource::getName)
@@ -300,7 +295,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     DataSegment koalaSegment = pollThenStopThenPublishKoalaSegment();
     Assert.assertEquals(
         ImmutableSet.of(koalaSegment),
-        ImmutableSet.copyOf(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(DS.KOALA).getSegments())
+        ImmutableSet.copyOf(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA).getSegments())
     );
   }
 
@@ -311,7 +306,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     DataSegment koalaSegment = pollThenStopThenPublishKoalaSegment();
     Assert.assertEquals(
         ImmutableSet.of(koalaSegment),
-        ImmutableSet.copyOf(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(DS.KOALA).getSegments())
+        ImmutableSet.copyOf(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA).getSegments())
     );
   }
 
@@ -350,10 +345,10 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
     Assert.assertFalse(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
     Assert.assertEquals(
-        ImmutableSet.of(DS.WIKI),
+        ImmutableSet.of(TestDataSource.WIKI),
         sqlSegmentsMetadataManager.retrieveAllDataSourceNames()
     );
-    final DataSegment koalaSegment = createNewSegment1(DS.KOALA);
+    final DataSegment koalaSegment = createNewSegment1(TestDataSource.KOALA);
     publishSegment(koalaSegment);
     sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
     return koalaSegment;
@@ -378,7 +373,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     Assert.assertEquals(
-        DS.WIKI,
+        TestDataSource.WIKI,
         Iterables.getOnlyElement(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments()).getName()
     );
   }
@@ -394,12 +389,12 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     allowUsedFlagLastUpdatedToBeNullable();
 
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-    int numChangedSegments = sqlSegmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(DS.WIKI);
+    int numChangedSegments = sqlSegmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(TestDataSource.WIKI);
     Assert.assertEquals(2, numChangedSegments);
 
     // Publish an unused segment with used_status_last_updated 2 hours ago
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -411,7 +406,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
 
     // Publish an unused segment with used_status_last_updated 2 days ago
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-16T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -423,7 +418,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
 
     // Publish an unused segment and set used_status_last_updated to null
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -433,7 +428,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         ImmutableList.of(wikiSegment2.getInterval()),
         sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
-            DS.WIKI,
+            TestDataSource.WIKI,
             null,
             DateTimes.of("3000"),
             1,
@@ -445,7 +440,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         ImmutableList.of(wikiSegment2.getInterval()),
         sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
-            DS.WIKI,
+            TestDataSource.WIKI,
             null,
             DateTimes.of(2012, 1, 7, 0, 0),
             1,
@@ -455,7 +450,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         ImmutableList.of(wikiSegment1.getInterval()),
         sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
-            DS.WIKI,
+            TestDataSource.WIKI,
             DateTimes.of(2012, 1, 7, 0, 0),
             DateTimes.of(2012, 4, 7, 0, 0),
             1,
@@ -465,7 +460,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         ImmutableList.of(),
         sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
-            DS.WIKI,
+            TestDataSource.WIKI,
             DateTimes.of(2012, 1, 7, 0, 0),
             DateTimes.of(2012, 1, 7, 0, 0),
             1,
@@ -476,7 +471,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         ImmutableList.of(wikiSegment2.getInterval(), wikiSegment1.getInterval()),
         sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
-            DS.WIKI,
+            TestDataSource.WIKI,
             null,
             DateTimes.of("3000"),
             5,
@@ -490,7 +485,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         ImmutableList.of(),
         sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
-            DS.WIKI,
+            TestDataSource.WIKI,
             DateTimes.COMPARE_DATE_AS_STRING_MIN,
             DateTimes.of("3000"),
             5,
@@ -504,7 +499,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         ImmutableList.of(koalaSegment2.getInterval()),
         sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
-            DS.KOALA,
+            TestDataSource.KOALA,
             DateTimes.COMPARE_DATE_AS_STRING_MIN,
             DateTimes.of("3000"),
             5,
@@ -520,13 +515,13 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     sqlSegmentsMetadataManager.poll();
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
-    publishSegment(createNewSegment1(DS.KOALA));
+    publishSegment(createNewSegment1(TestDataSource.KOALA));
 
-    awaitDataSourceAppeared(DS.KOALA);
-    int numChangedSegments = sqlSegmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(DS.KOALA);
+    awaitDataSourceAppeared(TestDataSource.KOALA);
+    int numChangedSegments = sqlSegmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(TestDataSource.KOALA);
     Assert.assertEquals(1, numChangedSegments);
-    awaitDataSourceDisappeared(DS.KOALA);
-    Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(DS.KOALA));
+    awaitDataSourceDisappeared(TestDataSource.KOALA);
+    Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA));
   }
 
   private static DataSegment createNewSegment1(String datasource)
@@ -555,18 +550,18 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     publishSegment(koalaSegment);
-    awaitDataSourceAppeared(DS.KOALA);
-    Assert.assertNotNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(DS.KOALA));
+    awaitDataSourceAppeared(TestDataSource.KOALA);
+    Assert.assertNotNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA));
 
     Assert.assertTrue(sqlSegmentsMetadataManager.markSegmentAsUnused(koalaSegment.getId()));
-    awaitDataSourceDisappeared(DS.KOALA);
-    Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(DS.KOALA));
+    awaitDataSourceDisappeared(TestDataSource.KOALA);
+    Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA));
   }
 
   private void awaitDataSourceAppeared(String datasource) throws InterruptedException
@@ -592,20 +587,20 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-16T20:19:12.565Z"
     );
 
     // Overshadowed by koalaSegment2
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -622,7 +617,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
         ImmutableSet.of(wikiSegment1, wikiSegment2),
         ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
     );
-    Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegments(DS.KOALA, segmentIds));
+    Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegments(TestDataSource.KOALA, segmentIds));
     sqlSegmentsMetadataManager.poll();
     Assert.assertEquals(
         ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2),
@@ -639,20 +634,20 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-16T20:19:12.565Z"
     );
 
     // Overshadowed by koalaSegment2
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -667,7 +662,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         2,
         sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             Intervals.ETERNITY,
             ImmutableList.of("2017-10-15T20:19:12.565Z", "2017-10-16T20:19:12.565Z")
         )
@@ -689,20 +684,20 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-16T20:19:12.565Z"
     );
 
     // Overshadowed by koalaSegment2
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -717,7 +712,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         0,
         sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             Intervals.of("2017/2018"),
             ImmutableList.of()
         )
@@ -739,20 +734,20 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-16T20:19:12.565Z"
     );
 
     // Overshadowed by koalaSegment2
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -767,7 +762,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         0,
         sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             Intervals.ETERNITY,
             ImmutableList.of()
         )
@@ -789,20 +784,20 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-16T20:19:12.565Z"
     );
 
     // Overshadowed by koalaSegment2
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -817,7 +812,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         2,
         sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             Intervals.of("2017-10-15/2017-10-18"),
             ImmutableList.of("2017-10-15T20:19:12.565Z", "2017-10-16T20:19:12.565Z")
         )
@@ -839,20 +834,20 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-16T20:19:12.565Z"
     );
 
     // Overshadowed by koalaSegment2
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -867,7 +862,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         0,
         sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             Intervals.ETERNITY,
             ImmutableList.of("foo", "bar")
         )
@@ -888,8 +883,8 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     sqlSegmentsMetadataManager.poll();
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
-    final DataSegment koalaSegment1 = createNewSegment1(DS.KOALA);
-    final DataSegment koalaSegment2 = createNewSegment1(DS.KOALA);
+    final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
+    final DataSegment koalaSegment2 = createNewSegment1(TestDataSource.KOALA);
 
     publishUnusedSegments(koalaSegment1, koalaSegment2);
     final ImmutableSet<String> segmentIds =
@@ -919,8 +914,8 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     sqlSegmentsMetadataManager.poll();
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
-    final DataSegment koalaSegment1 = createNewSegment1(DS.KOALA);
-    final DataSegment koalaSegment2 = createNewSegment1(DS.KOALA);
+    final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
+    final DataSegment koalaSegment2 = createNewSegment1(TestDataSource.KOALA);
 
     final ImmutableSet<String> segmentIds =
         ImmutableSet.of(koalaSegment1.getId().toString(), koalaSegment2.getId().toString());
@@ -933,7 +928,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     MatcherAssert.assertThat(
         Assert.assertThrows(
             DruidException.class,
-            () -> sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegments(DS.KOALA, segmentIds)
+            () -> sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegments(TestDataSource.KOALA, segmentIds)
         ),
         DruidExceptionMatcher
             .invalidInput()
@@ -949,20 +944,20 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     sqlSegmentsMetadataManager.poll();
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
-    final DataSegment koalaSegment1 = createNewSegment1(DS.KOALA);
+    final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-16T20:19:12.565Z"
     );
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     // Overshadowed by koalaSegment2
-    final DataSegment koalaSegment4 = createNewSegment2(DS.KOALA);
+    final DataSegment koalaSegment4 = createNewSegment2(TestDataSource.KOALA);
 
     publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3, koalaSegment4);
     final Interval theInterval = Intervals.of("2017-10-15T00:00:00.000/2017-10-18T00:00:00.000");
@@ -974,7 +969,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     );
 
     // 2 out of 3 segments match the interval
-    Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(DS.KOALA, theInterval, null));
+    Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(TestDataSource.KOALA, theInterval, null));
 
     sqlSegmentsMetadataManager.poll();
     Assert.assertEquals(
@@ -992,25 +987,25 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-16T20:19:12.565Z"
     );
 
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-19T00:00:00.000/2017-10-22T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
 
     // Overshadowed by koalaSegment2
-    final DataSegment koalaSegment4 = createNewSegment2(DS.KOALA);
+    final DataSegment koalaSegment4 = createNewSegment2(TestDataSource.KOALA);
 
     publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3, koalaSegment4);
     final Interval theInterval = Intervals.of("2017-10-16T00:00:00.000/2017-10-20T00:00:00.000");
@@ -1022,7 +1017,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     );
 
     // 1 out of 3 segments match the interval, other 2 overlap, only the segment fully contained will be marked unused
-    Assert.assertEquals(1, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(DS.KOALA, theInterval, null));
+    Assert.assertEquals(1, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(TestDataSource.KOALA, theInterval, null));
 
     sqlSegmentsMetadataManager.poll();
     Assert.assertEquals(
@@ -1039,8 +1034,8 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     sqlSegmentsMetadataManager.poll();
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
-    final DataSegment koalaSegment1 = createNewSegment1(DS.KOALA);
-    final DataSegment koalaSegment2 = createNewSegment1(DS.KOALA);
+    final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
+    final DataSegment koalaSegment2 = createNewSegment1(TestDataSource.KOALA);
 
     publishSegment(koalaSegment1);
     publishSegment(koalaSegment2);
@@ -1064,10 +1059,10 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     sqlSegmentsMetadataManager.poll();
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
-    final DataSegment koalaSegment1 = createNewSegment1(DS.KOALA);
-    final DataSegment koalaSegment2 = createNewSegment2(DS.KOALA);
+    final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
+    final DataSegment koalaSegment2 = createNewSegment2(TestDataSource.KOALA);
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -1078,7 +1073,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     final Interval theInterval = Intervals.of("2017-10-15T00:00:00.000/2017-10-18T00:00:00.000");
 
     // 2 out of 3 segments match the interval
-    Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(DS.KOALA, theInterval, null));
+    Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(TestDataSource.KOALA, theInterval, null));
 
     sqlSegmentsMetadataManager.poll();
     Assert.assertEquals(
@@ -1100,17 +1095,17 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     final String v2 = now.plus(Duration.standardDays(1)).toString();
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
         v1
     );
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         v2
     );
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
         v2
     );
@@ -1123,7 +1118,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         2,
         sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             theInterval,
             ImmutableList.of(v1, v2)
         )
@@ -1149,17 +1144,17 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     final String v2 = now.plus(Duration.standardDays(1)).toString();
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
         v1
     );
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         v2
     );
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
         v2
     );
@@ -1172,7 +1167,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         0,
         sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             theInterval,
             ImmutableList.of("foo", "bar", "baz")
         )
@@ -1198,17 +1193,17 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     final String v2 = now.plus(Duration.standardDays(1)).toString();
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
         v1
     );
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         v2
     );
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
         v2
     );
@@ -1221,7 +1216,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         0,
         sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             theInterval,
             ImmutableList.of()
         )
@@ -1247,17 +1242,17 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     final String v2 = now.plus(Duration.standardDays(1)).toString();
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
         v1
     );
     final DataSegment koalaSegment2 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         v2
     );
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
         v2
     );
@@ -1270,7 +1265,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertEquals(
         0,
         sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(
-            DS.KOALA,
+            TestDataSource.KOALA,
             theInterval,
             ImmutableList.of()
         )
@@ -1292,13 +1287,13 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
 
     final DataSegment koalaSegment1 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
-    final DataSegment koalaSegment2 = createNewSegment2(DS.KOALA);
+    final DataSegment koalaSegment2 = createNewSegment2(TestDataSource.KOALA);
     final DataSegment koalaSegment3 = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-19T00:00:00.000/2017-10-22T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -1309,7 +1304,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     final Interval theInterval = Intervals.of("2017-10-16T00:00:00.000/2017-10-20T00:00:00.000");
 
     // 1 out of 3 segments match the interval, other 2 overlap, only the segment fully contained will be marked unused
-    Assert.assertEquals(1, sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(DS.KOALA, theInterval, null));
+    Assert.assertEquals(1, sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(TestDataSource.KOALA, theInterval, null));
 
     sqlSegmentsMetadataManager.poll();
     Assert.assertEquals(
@@ -1348,14 +1343,14 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
     sqlSegmentsMetadataManager.start();
 
     Optional<Iterable<DataSegment>> segments = sqlSegmentsMetadataManager
-        .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(DS.WIKI, theInterval, true);
+        .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(TestDataSource.WIKI, theInterval, true);
     Assert.assertTrue(segments.isPresent());
     Set<DataSegment> dataSegmentSet = ImmutableSet.copyOf(segments.get());
     Assert.assertEquals(1, dataSegmentSet.size());
     Assert.assertTrue(dataSegmentSet.contains(wikiSegment1));
 
     final DataSegment wikiSegment3 = createSegment(
-        DS.WIKI,
+        TestDataSource.WIKI,
         "2012-03-16T00:00:00.000/2012-03-17T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
@@ -1363,7 +1358,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
 
     // New segment is not returned since we call without force poll
     segments = sqlSegmentsMetadataManager
-        .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(DS.WIKI, theInterval, false);
+        .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(TestDataSource.WIKI, theInterval, false);
     Assert.assertTrue(segments.isPresent());
     dataSegmentSet = ImmutableSet.copyOf(segments.get());
     Assert.assertEquals(1, dataSegmentSet.size());
@@ -1371,7 +1366,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
 
     // New segment is returned since we call with force poll
     segments = sqlSegmentsMetadataManager
-        .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(DS.WIKI, theInterval, true);
+        .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(TestDataSource.WIKI, theInterval, true);
     Assert.assertTrue(segments.isPresent());
     dataSegmentSet = ImmutableSet.copyOf(segments.get());
     Assert.assertEquals(2, dataSegmentSet.size());
@@ -1384,7 +1379,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
   {
     allowUsedFlagLastUpdatedToBeNullable();
     final DataSegment koalaSegment = createSegment(
-        DS.KOALA,
+        TestDataSource.KOALA,
         "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
         "2017-10-15T20:19:12.565Z"
     );
diff --git a/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java b/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java
index 5f583746266..18e7e276950 100644
--- a/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java
+++ b/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java
@@ -47,6 +47,7 @@ import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.rpc.HttpResponseException;
 import org.apache.druid.rpc.MockServiceClient;
 import org.apache.druid.rpc.RequestBuilder;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.MatcherAssert;
 import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
@@ -63,6 +64,7 @@ import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -451,4 +453,82 @@ public class OverlordClientImplTest
         overlordClient.taskPayload(taskID).get().getPayload()
     );
   }
+
+  @Test
+  public void test_isCompactionSupervisorEnabled()
+      throws JsonProcessingException, ExecutionException, InterruptedException
+  {
+    serviceClient.expectAndRespond(
+        new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/isSupervisorEnabled"),
+        HttpResponseStatus.OK,
+        Collections.emptyMap(),
+        DefaultObjectMapper.INSTANCE.writeValueAsBytes(false)
+    );
+
+    Assert.assertFalse(overlordClient.isCompactionSupervisorEnabled().get());
+  }
+
+  @Test
+  public void test_getCompactionSnapshots_nullDataSource()
+      throws JsonProcessingException, ExecutionException, InterruptedException
+  {
+    final List<AutoCompactionSnapshot> compactionSnapshots = Arrays.asList(
+        AutoCompactionSnapshot.builder("ds1")
+                              .withStatus(AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING)
+                              .build(),
+        AutoCompactionSnapshot.builder("ds2")
+                              .withStatus(AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING)
+                              .build()
+    );
+    serviceClient.expectAndRespond(
+        new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/status"),
+        HttpResponseStatus.OK,
+        Collections.emptyMap(),
+        DefaultObjectMapper.INSTANCE.writeValueAsBytes(compactionSnapshots)
+    );
+
+    Assert.assertEquals(
+        compactionSnapshots,
+        overlordClient.getCompactionSnapshots(null).get()
+    );
+  }
+
+  @Test
+  public void test_getCompactionSnapshots_nonNullDataSource()
+      throws JsonProcessingException, ExecutionException, InterruptedException
+  {
+    final List<AutoCompactionSnapshot> compactionSnapshots = Collections.singletonList(
+        AutoCompactionSnapshot.builder("ds1")
+                              .withStatus(AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING)
+                              .build()
+    );
+    serviceClient.expectAndRespond(
+        new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/status?dataSource=ds1"),
+        HttpResponseStatus.OK,
+        Collections.emptyMap(),
+        DefaultObjectMapper.INSTANCE.writeValueAsBytes(compactionSnapshots)
+    );
+
+    Assert.assertEquals(
+        compactionSnapshots,
+        overlordClient.getCompactionSnapshots("ds1").get()
+    );
+  }
+
+  @Test
+  public void test_getBytesAwaitingCompaction()
+      throws JsonProcessingException, ExecutionException, InterruptedException
+  {
+    serviceClient.expectAndRespond(
+        new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/progress?dataSource=ds1"),
+        HttpResponseStatus.OK,
+        Collections.emptyMap(),
+        DefaultObjectMapper.INSTANCE.writeValueAsBytes(100_000L)
+    );
+
+    Assert.assertEquals(
+        100_000L,
+        overlordClient.getBytesAwaitingCompaction("ds1").get().longValue()
+    );
+  }
 }
diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
new file mode 100644
index 00000000000..8b5c6bab6c4
--- /dev/null
+++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.druid.client.indexing.NoopOverlordClient;
+import org.apache.druid.client.indexing.TaskPayloadResponse;
+import org.apache.druid.indexer.TaskStatus;
+import org.apache.druid.indexer.TaskStatusPlus;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.CloseableIterators;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.parsers.CloseableIterator;
+import org.apache.druid.metadata.LockFilterPolicy;
+import org.apache.druid.server.coordinator.CreateDataSegments;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+import org.apache.druid.server.coordinator.DruidCompactionConfig;
+import org.apache.druid.server.coordinator.simulate.TestSegmentsMetadataManager;
+import org.apache.druid.timeline.DataSegment;
+import org.joda.time.Interval;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class CompactionRunSimulatorTest
+{
+  private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper();
+
+  private final CompactionRunSimulator simulator = new CompactionRunSimulator(
+      new CompactionStatusTracker(OBJECT_MAPPER),
+      new TestOverlordClient()
+  );
+
+  @Test
+  public void testSimulateClusterCompactionConfigUpdate()
+  {
+    final TestSegmentsMetadataManager segmentsMetadataManager = new TestSegmentsMetadataManager();
+
+    // Add some segments to the timeline
+    final List<DataSegment> wikiSegments
+        = CreateDataSegments.ofDatasource("wiki")
+                            .forIntervals(10, Granularities.DAY)
+                            .withNumPartitions(10)
+                            .startingAt("2013-01-01")
+                            .eachOfSizeInMb(100);
+    wikiSegments.forEach(segmentsMetadataManager::addSegment);
+
+    final CompactionSimulateResult simulateResult = simulator.simulateRunWithConfig(
+        DruidCompactionConfig.empty().withDatasourceConfig(
+            DataSourceCompactionConfig.builder().forDataSource("wiki").build()
+        ),
+        segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+                               .getUsedSegmentsTimelinesPerDataSource()
+    );
+
+    Assert.assertNotNull(simulateResult);
+
+    final Map<CompactionStatus.State, Table> compactionStates = simulateResult.getCompactionStates();
+    Assert.assertNotNull(compactionStates);
+
+    Assert.assertNull(compactionStates.get(CompactionStatus.State.COMPLETE));
+    Assert.assertNull(compactionStates.get(CompactionStatus.State.RUNNING));
+
+    final Table queuedTable = compactionStates.get(CompactionStatus.State.PENDING);
+    Assert.assertEquals(
+        Arrays.asList("dataSource", "interval", "numSegments", "bytes", "maxTaskSlots", "reasonToCompact"),
+        queuedTable.getColumnNames()
+    );
+    Assert.assertEquals(
+        Arrays.asList(
+            Arrays.asList("wiki", Intervals.of("2013-01-09/P1D"), 10, 1_000_000_000L, 1, "not compacted yet"),
+            Arrays.asList("wiki", Intervals.of("2013-01-08/P1D"), 10, 1_000_000_000L, 1, "not compacted yet"),
+            Arrays.asList("wiki", Intervals.of("2013-01-07/P1D"), 10, 1_000_000_000L, 1, "not compacted yet"),
+            Arrays.asList("wiki", Intervals.of("2013-01-06/P1D"), 10, 1_000_000_000L, 1, "not compacted yet"),
+            Arrays.asList("wiki", Intervals.of("2013-01-05/P1D"), 10, 1_000_000_000L, 1, "not compacted yet"),
+            Arrays.asList("wiki", Intervals.of("2013-01-04/P1D"), 10, 1_000_000_000L, 1, "not compacted yet"),
+            Arrays.asList("wiki", Intervals.of("2013-01-03/P1D"), 10, 1_000_000_000L, 1, "not compacted yet"),
+            Arrays.asList("wiki", Intervals.of("2013-01-02/P1D"), 10, 1_000_000_000L, 1, "not compacted yet"),
+            Arrays.asList("wiki", Intervals.of("2013-01-01/P1D"), 10, 1_000_000_000L, 1, "not compacted yet")
+        ),
+        queuedTable.getRows()
+    );
+
+    final Table skippedTable = compactionStates.get(CompactionStatus.State.SKIPPED);
+    Assert.assertEquals(
+        Arrays.asList("dataSource", "interval", "numSegments", "bytes", "reasonToSkip"),
+        skippedTable.getColumnNames()
+    );
+    Assert.assertEquals(
+        Collections.singletonList(
+            Arrays.asList("wiki", Intervals.of("2013-01-10/P1D"), 10, 1_000_000_000L, "skip offset from latest[P1D]")
+        ),
+        skippedTable.getRows()
+    );
+  }
+
+  private static class TestOverlordClient extends NoopOverlordClient
+  {
+    @Override
+    public ListenableFuture<CloseableIterator<TaskStatusPlus>> taskStatuses(
+        @Nullable String state,
+        @Nullable String dataSource,
+        @Nullable Integer maxCompletedTasks
+    )
+    {
+      return Futures.immediateFuture(CloseableIterators.withEmptyBaggage(Collections.emptyIterator()));
+    }
+
+    @Override
+    public ListenableFuture<Map<String, TaskStatus>> taskStatuses(Set<String> taskIds)
+    {
+      return Futures.immediateFuture(Collections.emptyMap());
+    }
+
+    @Override
+    public ListenableFuture<TaskPayloadResponse> taskPayload(String taskId)
+    {
+      return Futures.immediateFuture(null);
+    }
+
+    @Override
+    public ListenableFuture<Void> runTask(String taskId, Object taskObject)
+    {
+      return Futures.immediateVoidFuture();
+    }
+
+    @Override
+    public ListenableFuture<Void> cancelTask(String taskId)
+    {
+      return Futures.immediateVoidFuture();
+    }
+
+    @Override
+    public ListenableFuture<Map<String, List<Interval>>> findLockedIntervals(List<LockFilterPolicy> lockFilterPolicies)
+    {
+      return Futures.immediateFuture(Collections.emptyMap());
+    }
+  }
+}
diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java
new file mode 100644
index 00000000000..c4177f186c7
--- /dev/null
+++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig;
+import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
+import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.indexer.partitions.HashedPartitionsSpec;
+import org.apache.druid.indexer.partitions.PartitionsSpec;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.segment.data.CompressionStrategy;
+import org.apache.druid.segment.indexing.granularity.GranularitySpec;
+import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig;
+import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig;
+import org.apache.druid.timeline.CompactionState;
+import org.apache.druid.timeline.DataSegment;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+
+public class CompactionStatusTest
+{
+  private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper();
+
+  private static final DataSegment WIKI_SEGMENT
+      = DataSegment.builder()
+                   .dataSource(TestDataSource.WIKI)
+                   .interval(Intervals.of("2013-01-01/PT1H"))
+                   .size(100_000_000L)
+                   .version("v1")
+                   .build();
+
+  @Test
+  public void testFindPartitionsSpecWhenGivenIsNull()
+  {
+    final ClientCompactionTaskQueryTuningConfig tuningConfig
+        = ClientCompactionTaskQueryTuningConfig.from(null);
+    Assert.assertEquals(
+        new DynamicPartitionsSpec(null, Long.MAX_VALUE),
+        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
+    );
+  }
+
+  @Test
+  public void testFindPartitionsSpecWhenGivenIsDynamicWithNullMaxTotalRows()
+  {
+    final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, null);
+    final ClientCompactionTaskQueryTuningConfig tuningConfig
+        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
+    Assert.assertEquals(
+        new DynamicPartitionsSpec(null, Long.MAX_VALUE),
+        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
+    );
+  }
+
+  @Test
+  public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxTotalRows()
+  {
+    final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, 1000L);
+    final ClientCompactionTaskQueryTuningConfig tuningConfig
+        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
+    Assert.assertEquals(
+        partitionsSpec,
+        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
+    );
+  }
+
+  @Test
+  public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxRowsPerSegment()
+  {
+    final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(100, 1000L);
+    final ClientCompactionTaskQueryTuningConfig tuningConfig
+        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
+    Assert.assertEquals(
+        partitionsSpec,
+        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
+    );
+  }
+
+  @Test
+  public void testFindPartitionsSpecFromConfigWithDeprecatedMaxRowsPerSegmentAndMaxTotalRowsReturnGivenValues()
+  {
+    final DataSourceCompactionConfig config = new DataSourceCompactionConfig(
+        "datasource",
+        null,
+        null,
+        100,
+        null,
+        new UserCompactionTaskQueryTuningConfig(
+            null,
+            null,
+            null,
+            1000L,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        ),
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null
+    );
+    Assert.assertEquals(
+        new DynamicPartitionsSpec(100, 1000L),
+        CompactionStatus.findPartitionsSpecFromConfig(
+            ClientCompactionTaskQueryTuningConfig.from(config)
+        )
+    );
+  }
+
+  @Test
+  public void testFindPartitionsSpecWhenGivenIsHashed()
+  {
+    final PartitionsSpec partitionsSpec =
+        new HashedPartitionsSpec(null, 100, Collections.singletonList("dim"));
+    final ClientCompactionTaskQueryTuningConfig tuningConfig
+        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
+    Assert.assertEquals(
+        partitionsSpec,
+        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
+    );
+  }
+
+  @Test
+  public void testFindPartitionsSpecWhenGivenIsRange()
+  {
+    final PartitionsSpec partitionsSpec =
+        new DimensionRangePartitionsSpec(null, 10000, Collections.singletonList("dim"), false);
+    final ClientCompactionTaskQueryTuningConfig tuningConfig
+        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
+    Assert.assertEquals(
+        partitionsSpec,
+        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
+    );
+  }
+
+  @Test
+  public void testStatusWhenLastCompactionStateIsNull()
+  {
+    verifyCompactionStatusIsPendingBecause(
+        null,
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
+        "not compacted yet"
+    );
+  }
+
+  @Test
+  public void testStatusWhenLastCompactionStateIsEmpty()
+  {
+    verifyCompactionStatusIsPendingBecause(
+        new CompactionState(null, null, null, null, null, null),
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
+        "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows], current[null]"
+    );
+  }
+
+  @Test
+  public void testStatusOnPartitionsSpecMismatch()
+  {
+    final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, 0L);
+
+    final CompactionState lastCompactionState
+        = new CompactionState(currentPartitionsSpec, null, null, null, null, null);
+    final DataSourceCompactionConfig compactionConfig
+        = DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build();
+
+    verifyCompactionStatusIsPendingBecause(
+        lastCompactionState,
+        compactionConfig,
+        "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows],"
+        + " current['dynamic' with 100 rows]"
+    );
+  }
+
+  @Test
+  public void testStatusOnIndexSpecMismatch()
+  {
+    final IndexSpec currentIndexSpec
+        = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build();
+
+    final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, 0L);
+    final CompactionState lastCompactionState = new CompactionState(
+        currentPartitionsSpec,
+        null,
+        null,
+        null,
+        currentIndexSpec.asMap(OBJECT_MAPPER),
+        null
+    );
+    final DataSourceCompactionConfig compactionConfig = DataSourceCompactionConfig
+        .builder()
+        .forDataSource(TestDataSource.WIKI)
+        .withTuningConfig(createTuningConfig(currentPartitionsSpec, null))
+        .build();
+
+    verifyCompactionStatusIsPendingBecause(
+        lastCompactionState,
+        compactionConfig,
+        "'indexSpec' mismatch: "
+        + "required[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{},"
+        + " dimensionCompression=lz4, stringDictionaryEncoding=Utf8{},"
+        + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null,"
+        + " jsonCompression=null, segmentLoader=null}], "
+        + "current[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{},"
+        + " dimensionCompression=zstd, stringDictionaryEncoding=Utf8{},"
+        + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null,"
+        + " jsonCompression=null, segmentLoader=null}]"
+    );
+  }
+
+  @Test
+  public void testStatusOnSegmentGranularityMismatch()
+  {
+    final GranularitySpec currentGranularitySpec
+        = new UniformGranularitySpec(Granularities.HOUR, null, null);
+
+    final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, 0L);
+    final IndexSpec currentIndexSpec
+        = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build();
+    final CompactionState lastCompactionState = new CompactionState(
+        currentPartitionsSpec,
+        null,
+        null,
+        null,
+        currentIndexSpec.asMap(OBJECT_MAPPER),
+        currentGranularitySpec.asMap(OBJECT_MAPPER)
+    );
+    final DataSourceCompactionConfig compactionConfig = DataSourceCompactionConfig
+        .builder()
+        .forDataSource(TestDataSource.WIKI)
+        .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec))
+        .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null))
+        .build();
+
+    verifyCompactionStatusIsPendingBecause(
+        lastCompactionState,
+        compactionConfig,
+        "'segmentGranularity' mismatch: required[DAY], current[HOUR]"
+    );
+  }
+
+  @Test
+  public void testStatusWhenLastCompactionStateSameAsRequired()
+  {
+    final GranularitySpec currentGranularitySpec
+        = new UniformGranularitySpec(Granularities.HOUR, null, null);
+    final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, 0L);
+    final IndexSpec currentIndexSpec
+        = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build();
+    final CompactionState lastCompactionState = new CompactionState(
+        currentPartitionsSpec,
+        null,
+        null,
+        null,
+        currentIndexSpec.asMap(OBJECT_MAPPER),
+        currentGranularitySpec.asMap(OBJECT_MAPPER)
+    );
+    final DataSourceCompactionConfig compactionConfig = DataSourceCompactionConfig
+        .builder()
+        .forDataSource(TestDataSource.WIKI)
+        .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec))
+        .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null))
+        .build();
+
+    final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build();
+    final CompactionStatus status = CompactionStatus.compute(
+        CompactionCandidate.from(Collections.singletonList(segment)),
+        compactionConfig,
+        OBJECT_MAPPER
+    );
+    Assert.assertTrue(status.isComplete());
+  }
+
+  private void verifyCompactionStatusIsPendingBecause(
+      CompactionState lastCompactionState,
+      DataSourceCompactionConfig compactionConfig,
+      String expectedReason
+  )
+  {
+    final DataSegment segment
+        = DataSegment.builder(WIKI_SEGMENT)
+                     .lastCompactionState(lastCompactionState)
+                     .build();
+    final CompactionStatus status = CompactionStatus.compute(
+        CompactionCandidate.from(Collections.singletonList(segment)),
+        compactionConfig,
+        OBJECT_MAPPER
+    );
+
+    Assert.assertFalse(status.isComplete());
+    Assert.assertEquals(expectedReason, status.getReason());
+  }
+
+  private static DataSourceCompactionConfig createCompactionConfig(
+      PartitionsSpec partitionsSpec
+  )
+  {
+    return DataSourceCompactionConfig.builder()
+                                     .forDataSource(TestDataSource.WIKI)
+                                     .withTuningConfig(createTuningConfig(partitionsSpec, null))
+                                     .build();
+  }
+
+  private static UserCompactionTaskQueryTuningConfig createTuningConfig(
+      PartitionsSpec partitionsSpec,
+      IndexSpec indexSpec
+  )
+  {
+    return new UserCompactionTaskQueryTuningConfig(
+        null,
+        null, null, null, null, partitionsSpec, indexSpec, null, null,
+        null, null, null, null, null, null, null, null, null, null
+    );
+  }
+}
diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java
new file mode 100644
index 00000000000..f64c0865d6d
--- /dev/null
+++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.druid.client.indexing.ClientCompactionTaskQuery;
+import org.apache.druid.indexer.TaskState;
+import org.apache.druid.indexer.TaskStatus;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.server.coordinator.CreateDataSegments;
+import org.apache.druid.timeline.DataSegment;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collections;
+
+public class CompactionStatusTrackerTest
+{
+  private static final ObjectMapper MAPPER = new DefaultObjectMapper();
+  private static final DataSegment WIKI_SEGMENT
+      = CreateDataSegments.ofDatasource(TestDataSource.WIKI).eachOfSizeInMb(100).get(0);
+
+  private CompactionStatusTracker statusTracker;
+
+  @Before
+  public void setup()
+  {
+    statusTracker = new CompactionStatusTracker(MAPPER);
+  }
+
+  @Test
+  public void testGetLatestTaskStatusForSubmittedTask()
+  {
+    final CompactionCandidate candidateSegments
+        = CompactionCandidate.from(Collections.singletonList(WIKI_SEGMENT));
+    statusTracker.onTaskSubmitted(createCompactionTask("task1"), candidateSegments);
+
+    CompactionTaskStatus status = statusTracker.getLatestTaskStatus(candidateSegments);
+    Assert.assertEquals(TaskState.RUNNING, status.getState());
+  }
+
+  @Test
+  public void testGetLatestTaskStatusForSuccessfulTask()
+  {
+    final CompactionCandidate candidateSegments
+        = CompactionCandidate.from(Collections.singletonList(WIKI_SEGMENT));
+    statusTracker.onTaskSubmitted(createCompactionTask("task1"), candidateSegments);
+    statusTracker.onTaskFinished("task1", TaskStatus.success("task1"));
+
+    CompactionTaskStatus status = statusTracker.getLatestTaskStatus(candidateSegments);
+    Assert.assertEquals(TaskState.SUCCESS, status.getState());
+  }
+
+  @Test
+  public void testGetLatestTaskStatusForFailedTask()
+  {
+    final CompactionCandidate candidateSegments
+        = CompactionCandidate.from(Collections.singletonList(WIKI_SEGMENT));
+    statusTracker.onTaskSubmitted(createCompactionTask("task1"), candidateSegments);
+    statusTracker.onTaskFinished("task1", TaskStatus.failure("task1", "some failure"));
+
+    CompactionTaskStatus status = statusTracker.getLatestTaskStatus(candidateSegments);
+    Assert.assertEquals(TaskState.FAILED, status.getState());
+    Assert.assertEquals(1, status.getNumConsecutiveFailures());
+  }
+
+  @Test
+  public void testGetLatestTaskStatusForRepeatedlyFailingTask()
+  {
+    final CompactionCandidate candidateSegments
+        = CompactionCandidate.from(Collections.singletonList(WIKI_SEGMENT));
+
+    statusTracker.onTaskSubmitted(createCompactionTask("task1"), candidateSegments);
+    statusTracker.onTaskFinished("task1", TaskStatus.failure("task1", "some failure"));
+
+    statusTracker.onTaskSubmitted(createCompactionTask("task2"), candidateSegments);
+    CompactionTaskStatus status = statusTracker.getLatestTaskStatus(candidateSegments);
+    Assert.assertEquals(TaskState.RUNNING, status.getState());
+    Assert.assertEquals(1, status.getNumConsecutiveFailures());
+
+    statusTracker.onTaskFinished("task2", TaskStatus.failure("task2", "second failure"));
+
+    status = statusTracker.getLatestTaskStatus(candidateSegments);
+    Assert.assertEquals(TaskState.FAILED, status.getState());
+    Assert.assertEquals(2, status.getNumConsecutiveFailures());
+  }
+
+  private ClientCompactionTaskQuery createCompactionTask(
+      String taskId
+  )
+  {
+    return new ClientCompactionTaskQuery(
+        taskId,
+        TestDataSource.WIKI,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null
+    );
+  }
+}
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/compact/DataSourceCompactibleSegmentIteratorTest.java b/server/src/test/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIteratorTest.java
similarity index 88%
rename from server/src/test/java/org/apache/druid/server/coordinator/compact/DataSourceCompactibleSegmentIteratorTest.java
rename to server/src/test/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIteratorTest.java
index a2765ccfc09..eea081b9f51 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/compact/DataSourceCompactibleSegmentIteratorTest.java
+++ b/server/src/test/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIteratorTest.java
@@ -17,14 +17,12 @@
  * under the License.
  */
 
-package org.apache.druid.server.coordinator.compact;
+package org.apache.druid.server.compaction;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
-import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.joda.time.Interval;
-import org.joda.time.Period;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -62,7 +60,7 @@ public class DataSourceCompactibleSegmentIteratorTest
         Intervals.of("2018-12-24/2018-12-25"),
         Intervals.of("2018-12-29/2019-01-01")
     );
-    final List<Interval> fullSkipIntervals = DataSourceCompactibleSegmentIterator.sortAndAddSkipIntervalFromLatest(
+    /*final List<Interval> fullSkipIntervals = DataSourceCompactibleSegmentIterator.sortAndAddSkipIntervalFromLatest(
         DateTimes.of("2019-01-01"),
         new Period(72, 0, 0, 0),
         null,
@@ -70,8 +68,8 @@ public class DataSourceCompactibleSegmentIteratorTest
             Intervals.of("2018-12-30/2018-12-31"),
             Intervals.of("2018-12-24/2018-12-25")
         )
-    );
+    );*/
 
-    Assert.assertEquals(expectedIntervals, fullSkipIntervals);
+    //Assert.assertEquals(expectedIntervals, fullSkipIntervals);
   }
 }
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/compact/NewestSegmentFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java
similarity index 52%
rename from server/src/test/java/org/apache/druid/server/coordinator/compact/NewestSegmentFirstPolicyTest.java
rename to server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java
index 8f24a4ebb7e..7580582685b 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/compact/NewestSegmentFirstPolicyTest.java
+++ b/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java
@@ -17,12 +17,11 @@
  * under the License.
  */
 
-package org.apache.druid.server.coordinator.compact;
+package org.apache.druid.server.compaction;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.InjectableValues;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -34,6 +33,7 @@ import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.granularity.PeriodGranularity;
 import org.apache.druid.java.util.common.guava.Comparators;
 import org.apache.druid.math.expr.ExprMacroTable;
@@ -43,9 +43,11 @@ import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.query.expression.TestExprMacroTable;
 import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.segment.data.ConciseBitmapSerdeFactory;
 import org.apache.druid.segment.incremental.OnheapIncrementalIndex;
 import org.apache.druid.segment.transform.TransformSpec;
+import org.apache.druid.server.coordinator.CreateDataSegments;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
 import org.apache.druid.server.coordinator.UserCompactionTaskDimensionsConfig;
 import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig;
@@ -56,17 +58,16 @@ import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.Partitions;
 import org.apache.druid.timeline.SegmentTimeline;
 import org.apache.druid.timeline.partition.NumberedShardSpec;
-import org.apache.druid.timeline.partition.ShardSpec;
 import org.apache.druid.timeline.partition.TombstoneShardSpec;
 import org.apache.druid.utils.Streams;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Interval;
 import org.joda.time.Period;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -78,26 +79,31 @@ import java.util.stream.Collectors;
 
 public class NewestSegmentFirstPolicyTest
 {
-  private static final String DATA_SOURCE = "dataSource";
-  private static final long DEFAULT_SEGMENT_SIZE = 1000;
   private static final int DEFAULT_NUM_SEGMENTS_PER_SHARD = 4;
   private final ObjectMapper mapper = new DefaultObjectMapper();
-  private final NewestSegmentFirstPolicy policy = new NewestSegmentFirstPolicy(mapper);
+  private final NewestSegmentFirstPolicy policy = new NewestSegmentFirstPolicy(null);
+  private CompactionStatusTracker statusTracker;
+
+  @Before
+  public void setup()
+  {
+    statusTracker = new CompactionStatusTracker(mapper);
+  }
 
   @Test
   public void testLargeOffsetAndSmallSegmentInterval()
   {
-    final Period segmentPeriod = new Period("PT1H");
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(10000, new Period("P2D"), null)),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            createTimeline(
-                new SegmentGenerateSpec(Intervals.of("2017-11-16T20:00:00/2017-11-17T04:00:00"), segmentPeriod),
-                new SegmentGenerateSpec(Intervals.of("2017-11-14T00:00:00/2017-11-16T07:00:00"), segmentPeriod)
-            )
-        ),
-        Collections.emptyMap()
+    final Period segmentPeriod = Period.hours(1);
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withSkipOffsetFromLatest(new Period("P2D")).build(),
+        createTimeline(
+            createSegments().forIntervals(8, Granularities.HOUR)
+                            .startingAt("2017-11-16T20:00:00Z")
+                            .withNumPartitions(4),
+            createSegments().forIntervals(55, Granularities.HOUR)
+                            .startingAt("2017-11-14")
+                            .withNumPartitions(4)
+        )
     );
 
     assertCompactSegmentIntervals(
@@ -112,17 +118,17 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testSmallOffsetAndLargeSegmentInterval()
   {
-    final Period segmentPeriod = new Period("PT1H");
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(10000, new Period("PT1M"), null)),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            createTimeline(
-                new SegmentGenerateSpec(Intervals.of("2017-11-16T20:00:00/2017-11-17T04:00:00"), segmentPeriod),
-                new SegmentGenerateSpec(Intervals.of("2017-11-14T00:00:00/2017-11-16T07:00:00"), segmentPeriod)
-            )
-        ),
-        Collections.emptyMap()
+    final Period segmentPeriod = Period.hours(1);
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withSkipOffsetFromLatest(new Period("PT1M")).build(),
+        createTimeline(
+            createSegments().forIntervals(8, Granularities.HOUR)
+                            .startingAt("2017-11-16T20:00:00Z")
+                            .withNumPartitions(4),
+            createSegments().forIntervals(55, Granularities.HOUR)
+                            .startingAt("2017-11-14")
+                            .withNumPartitions(4)
+        )
     );
 
     assertCompactSegmentIntervals(
@@ -145,18 +151,17 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testLargeGapInData()
   {
-    final Period segmentPeriod = new Period("PT1H");
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(10000, new Period("PT1H1M"), null)),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            createTimeline(
-                new SegmentGenerateSpec(Intervals.of("2017-11-16T20:00:00/2017-11-17T04:00:00"), segmentPeriod),
-                // larger gap than SegmentCompactionUtil.LOOKUP_PERIOD (1 day)
-                new SegmentGenerateSpec(Intervals.of("2017-11-14T00:00:00/2017-11-15T07:00:00"), segmentPeriod)
-            )
-        ),
-        Collections.emptyMap()
+    final Period segmentPeriod = Period.hours(1);
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withSkipOffsetFromLatest(Period.minutes(61)).build(),
+        createTimeline(
+            createSegments().forIntervals(8, Granularities.HOUR)
+                            .startingAt("2017-11-16T20:00:00Z")
+                            .withNumPartitions(4),
+            createSegments().forIntervals(31, Granularities.HOUR)
+                            .startingAt("2017-11-14")
+                            .withNumPartitions(4)
+        )
     );
 
     assertCompactSegmentIntervals(
@@ -179,32 +184,22 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testHugeShard()
   {
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(10000, new Period("P1D"), null)),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            createTimeline(
-                new SegmentGenerateSpec(
-                    Intervals.of("2017-11-17T00:00:00/2017-11-18T03:00:00"),
-                    new Period("PT1H"),
-                    200,
-                    DEFAULT_NUM_SEGMENTS_PER_SHARD
-                ),
-                new SegmentGenerateSpec(
-                    Intervals.of("2017-11-09T00:00:00/2017-11-17T00:00:00"),
-                    new Period("P2D"),
-                    13000, // larger than target compact segment size
-                    1
-                ),
-                new SegmentGenerateSpec(
-                    Intervals.of("2017-11-05T00:00:00/2017-11-09T00:00:00"),
-                    new Period("PT1H"),
-                    200,
-                    DEFAULT_NUM_SEGMENTS_PER_SHARD
-                )
-            )
-        ),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withSkipOffsetFromLatest(Period.days(1)).build(),
+        createTimeline(
+            createSegments()
+                .forIntervals(27, Granularities.HOUR)
+                .startingAt("2017-11-17")
+                .withNumPartitions(4),
+            createSegments()
+                .forIntervals(4, new PeriodGranularity(Period.days(2), null, null))
+                .startingAt("2017-11-09")
+                .withNumPartitions(1),
+            createSegments()
+                .forIntervals(96, Granularities.HOUR)
+                .startingAt("2017-11-05")
+                .withNumPartitions(4)
+        )
     );
 
     Interval lastInterval = null;
@@ -229,38 +224,22 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testManySegmentsPerShard()
   {
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(800000, new Period("P1D"), null)),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            createTimeline(
-                new SegmentGenerateSpec(
-                    Intervals.of("2017-12-04T01:00:00/2017-12-05T03:00:00"),
-                    new Period("PT1H"),
-                    375,
-                    80
-                ),
-                new SegmentGenerateSpec(
-                    Intervals.of("2017-12-04T00:00:00/2017-12-04T01:00:00"),
-                    new Period("PT1H"),
-                    200,
-                    150
-                ),
-                new SegmentGenerateSpec(
-                    Intervals.of("2017-12-03T18:00:00/2017-12-04T00:00:00"),
-                    new Period("PT6H"),
-                    200000,
-                    1
-                ),
-                new SegmentGenerateSpec(
-                    Intervals.of("2017-12-03T11:00:00/2017-12-03T18:00:00"),
-                    new Period("PT1H"),
-                    375,
-                    80
-                )
-            )
-        ),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withSkipOffsetFromLatest(Period.days(1)).build(),
+        createTimeline(
+            createSegments().forIntervals(26, Granularities.HOUR)
+                            .startingAt("2017-12-04T01:00:00Z")
+                            .withNumPartitions(80),
+            createSegments().forIntervals(1, Granularities.HOUR)
+                            .startingAt("2017-12-04")
+                            .withNumPartitions(150),
+            createSegments().forIntervals(1, Granularities.SIX_HOUR)
+                            .startingAt("2017-12-03T18:00:00Z")
+                            .withNumPartitions(1),
+            createSegments().forIntervals(7, Granularities.HOUR)
+                            .startingAt("2017-12-03T11:00:00Z")
+                            .withNumPartitions(80)
+        )
     );
 
     Interval lastInterval = null;
@@ -283,25 +262,30 @@ public class NewestSegmentFirstPolicyTest
   }
 
   @Test
-  public void testSkipUnknownDataSource()
+  public void testSkipDataSourceWithNoSegments()
   {
-    final String unknownDataSource = "unknown";
-    final Period segmentPeriod = new Period("PT1H");
-    final CompactionSegmentIterator iterator = policy.createIterator(
+    final Period segmentPeriod = Period.hours(1);
+    final CompactionSegmentIterator iterator = new PriorityBasedCompactionSegmentIterator(
+        policy,
         ImmutableMap.of(
-            unknownDataSource,
-            createCompactionConfig(10000, new Period("P2D"), null),
-            DATA_SOURCE,
-            createCompactionConfig(10000, new Period("P2D"), null)
+            TestDataSource.KOALA,
+            configBuilder().forDataSource(TestDataSource.KOALA).build(),
+            TestDataSource.WIKI,
+            configBuilder().forDataSource(TestDataSource.WIKI).withSkipOffsetFromLatest(Period.days(2)).build()
         ),
         ImmutableMap.of(
-            DATA_SOURCE,
+            TestDataSource.WIKI,
             createTimeline(
-                new SegmentGenerateSpec(Intervals.of("2017-11-16T20:00:00/2017-11-17T04:00:00"), segmentPeriod),
-                new SegmentGenerateSpec(Intervals.of("2017-11-14T00:00:00/2017-11-16T07:00:00"), segmentPeriod)
+                createSegments().forIntervals(8, Granularities.HOUR)
+                                .startingAt("2017-11-16T20:00:00Z")
+                                .withNumPartitions(4),
+                createSegments().forIntervals(55, Granularities.HOUR)
+                                .startingAt("2017-11-14")
+                                .withNumPartitions(4)
             )
         ),
-        Collections.emptyMap()
+        Collections.emptyMap(),
+        statusTracker
     );
 
     assertCompactSegmentIntervals(
@@ -316,31 +300,32 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testClearSegmentsToCompactWhenSkippingSegments()
   {
-    final long inputSegmentSizeBytes = 800000;
-    final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-12-03T00:00:00/2017-12-04T00:00:00"),
-            new Period("P1D"),
-            inputSegmentSizeBytes / 2 + 10,
-            1
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-12-02T00:00:00/2017-12-03T00:00:00"),
-            new Period("P1D"),
-            inputSegmentSizeBytes + 10, // large segment
-            1
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-12-01T00:00:00/2017-12-02T00:00:00"),
-            new Period("P1D"),
-            inputSegmentSizeBytes / 3 + 10,
-            2
-        )
+    final long inputSegmentSizeBytes = 800_000;
+    final List<DataSegment> segments = new ArrayList<>(
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-12-03")
+            .withNumPartitions(1)
+            .eachOfSize(inputSegmentSizeBytes / 2 + 10)
     );
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(inputSegmentSizeBytes, new Period("P0D"), null)),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    segments.addAll(
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-12-02")
+            .withNumPartitions(1)
+            .eachOfSize(inputSegmentSizeBytes + 10) // large segment
+    );
+    segments.addAll(
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-12-01")
+            .withNumPartitions(2)
+            .eachOfSize(inputSegmentSizeBytes / 3 + 10)
+    );
+    final SegmentTimeline timeline = SegmentTimeline.forSegments(segments);
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withInputSegmentSizeBytes(inputSegmentSizeBytes).build(),
+        timeline
     );
 
     final List<DataSegment> expectedSegmentsToCompact = new ArrayList<>(
@@ -354,11 +339,11 @@ public class NewestSegmentFirstPolicyTest
     expectedSegmentsToCompact2.sort(Comparator.naturalOrder());
 
     Set<List<DataSegment>> observedSegments = Streams.sequentialStreamFrom(iterator)
-                                                     .map(SegmentsToCompact::getSegments)
+                                                     .map(CompactionCandidate::getSegments)
                                                      .collect(Collectors.toSet());
     Assert.assertEquals(
-        observedSegments,
-        ImmutableSet.of(expectedSegmentsToCompact, expectedSegmentsToCompact2)
+        ImmutableSet.of(expectedSegmentsToCompact, expectedSegmentsToCompact2),
+        observedSegments
     );
   }
 
@@ -366,18 +351,15 @@ public class NewestSegmentFirstPolicyTest
   public void testIfFirstSegmentIsInSkipOffset()
   {
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-12-02T14:00:00/2017-12-03T00:00:00"),
-            new Period("PT5H"),
-            40000,
-            1
-        )
+        createSegments()
+            .forIntervals(2, new PeriodGranularity(Period.hours(5), null, null))
+            .startingAt("2017-12-02T14:00:00Z")
+            .withNumPartitions(1)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(40000, new Period("P1D"), null)),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withSkipOffsetFromLatest(Period.days(1)).build(),
+        timeline
     );
 
     Assert.assertFalse(iterator.hasNext());
@@ -387,18 +369,15 @@ public class NewestSegmentFirstPolicyTest
   public void testIfFirstSegmentOverlapsSkipOffset()
   {
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-12-01T23:00:00/2017-12-03T00:00:00"),
-            new Period("PT5H"),
-            40000,
-            1
-        )
+        createSegments()
+            .forIntervals(5, new PeriodGranularity(Period.hours(5), null, null))
+            .startingAt("2017-12-01T23:00:00Z")
+            .withNumPartitions(1)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(40000, new Period("P1D"), null)),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withSkipOffsetFromLatest(Period.days(1)).build(),
+        timeline
     );
 
     Assert.assertFalse(iterator.hasNext());
@@ -408,19 +387,25 @@ public class NewestSegmentFirstPolicyTest
   public void testIfSegmentsSkipOffsetWithConfiguredSegmentGranularityEqual()
   {
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(Intervals.of("2017-11-30T23:00:00/2017-12-03T00:00:00"), new Period("P1D")),
-        new SegmentGenerateSpec(Intervals.of("2017-10-14T00:00:00/2017-10-15T00:00:00"), new Period("P1D"))
+        createSegments().forIntervals(2, Granularities.DAY)
+                        .startingAt("2017-12-01")
+                        .withNumPartitions(4),
+        createSegments().forIntervals(1, Granularities.DAY)
+                        .startingAt("2017-10-14")
+                        .withNumPartitions(4)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(40000, new Period("P1D"), new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder()
+            .withSkipOffsetFromLatest(Period.days(1))
+            .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null))
+            .build(),
+        timeline
     );
 
     // We should only get segments in Oct
     final List<DataSegment> expectedSegmentsToCompact = new ArrayList<>(
-        timeline.findNonOvershadowedObjectsInInterval(Intervals.of("2017-10-14T00:00:00/2017-12-02T00:00:00"), Partitions.ONLY_COMPLETE)
+        timeline.findNonOvershadowedObjectsInInterval(Intervals.of("2017-10-14/2017-12-02"), Partitions.ONLY_COMPLETE)
     );
 
     Assert.assertTrue(iterator.hasNext());
@@ -441,19 +426,25 @@ public class NewestSegmentFirstPolicyTest
         // - Cross between month boundary of latest month (starts in Nov and ends in Dec). This should be skipped
         // - Fully in latest month (starts in Dec and ends in Dec). This should be skipped
         // - Does not overlap latest month (starts in Oct and ends in Oct). This should not be skipped
-        new SegmentGenerateSpec(Intervals.of("2017-11-30T23:00:00/2017-12-03T00:00:00"), new Period("PT5H")),
-        new SegmentGenerateSpec(Intervals.of("2017-10-14T00:00:00/2017-10-15T00:00:00"), new Period("PT5H"))
+        createSegments().forIntervals(5, new PeriodGranularity(Period.hours(5), null, null))
+                        .startingAt("2017-11-30T23:00:00Z")
+                        .withNumPartitions(4),
+        createSegments().forIntervals(4, new PeriodGranularity(Period.hours(5), null, null))
+                        .startingAt("2017-10-14T04:00:00Z")
+                        .withNumPartitions(4)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(40000, new Period("P1D"), new UserCompactionTaskGranularityConfig(Granularities.MONTH, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder()
+            .withSkipOffsetFromLatest(Period.days(1))
+            .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.MONTH, null, null))
+            .build(),
+        timeline
     );
 
     // We should only get segments in Oct
     final List<DataSegment> expectedSegmentsToCompact = new ArrayList<>(
-        timeline.findNonOvershadowedObjectsInInterval(Intervals.of("2017-10-14T00:00:00/2017-10-15T00:00:00"), Partitions.ONLY_COMPLETE)
+        timeline.findNonOvershadowedObjectsInInterval(Intervals.of("2017-10-14/P1D"), Partitions.ONLY_COMPLETE)
     );
 
     Assert.assertTrue(iterator.hasNext());
@@ -467,14 +458,20 @@ public class NewestSegmentFirstPolicyTest
   public void testIfSegmentsSkipOffsetWithConfiguredSegmentGranularitySmaller()
   {
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(Intervals.of("2017-12-01T23:00:00/2017-12-03T00:00:00"), new Period("PT5H")),
-        new SegmentGenerateSpec(Intervals.of("2017-10-14T00:00:00/2017-10-15T00:00:00"), new Period("PT5H"))
+        createSegments().forIntervals(5, new PeriodGranularity(Period.hours(5), null, null))
+                        .startingAt("2017-12-01T23:00:00Z")
+                        .withNumPartitions(4),
+        createSegments().forIntervals(4, new PeriodGranularity(Period.hours(5), null, null))
+                        .startingAt("2017-10-14")
+                        .withNumPartitions(4)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(40000, new Period("P1D"), new UserCompactionTaskGranularityConfig(Granularities.MINUTE, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder()
+            .withSkipOffsetFromLatest(Period.days(1))
+            .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.MINUTE, null, null))
+            .build(),
+        timeline
     );
 
     // We should only get segments in Oct
@@ -495,24 +492,30 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testWithSkipIntervals()
   {
-    final Period segmentPeriod = new Period("PT1H");
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(10000, new Period("P1D"), null)),
+    final Period segmentPeriod = Period.hours(1);
+    final CompactionSegmentIterator iterator = new PriorityBasedCompactionSegmentIterator(
+        policy,
+        ImmutableMap.of(TestDataSource.WIKI, configBuilder().withSkipOffsetFromLatest(Period.days(1)).build()),
         ImmutableMap.of(
-            DATA_SOURCE,
+            TestDataSource.WIKI,
             createTimeline(
-                new SegmentGenerateSpec(Intervals.of("2017-11-16T20:00:00/2017-11-17T04:00:00"), segmentPeriod),
-                new SegmentGenerateSpec(Intervals.of("2017-11-14T00:00:00/2017-11-16T07:00:00"), segmentPeriod)
+                createSegments().forIntervals(8, Granularities.HOUR)
+                                .startingAt("2017-11-16T20:00:00Z")
+                                .withNumPartitions(4),
+                createSegments().forIntervals(55, Granularities.HOUR)
+                                .startingAt("2017-11-14")
+                                .withNumPartitions(4)
             )
         ),
         ImmutableMap.of(
-            DATA_SOURCE,
+            TestDataSource.WIKI,
             ImmutableList.of(
                 Intervals.of("2017-11-16T00:00:00/2017-11-17T00:00:00"),
                 Intervals.of("2017-11-15T00:00:00/2017-11-15T20:00:00"),
                 Intervals.of("2017-11-13T00:00:00/2017-11-14T01:00:00")
             )
-        )
+        ),
+        statusTracker
     );
 
     assertCompactSegmentIntervals(
@@ -535,22 +538,24 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testHoleInSearchInterval()
   {
-    final Period segmentPeriod = new Period("PT1H");
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(10000, new Period("PT1H"), null)),
+    final Period segmentPeriod = Period.hours(1);
+    final CompactionSegmentIterator iterator = new PriorityBasedCompactionSegmentIterator(
+        policy,
+        ImmutableMap.of(TestDataSource.WIKI, configBuilder().withSkipOffsetFromLatest(Period.hours(1)).build()),
         ImmutableMap.of(
-            DATA_SOURCE,
+            TestDataSource.WIKI,
             createTimeline(
-                new SegmentGenerateSpec(Intervals.of("2017-11-16T00:00:00/2017-11-17T00:00:00"), segmentPeriod)
+                createSegments().forIntervals(1, Granularities.HOUR).startingAt("2017-11-16").withNumPartitions(4)
             )
         ),
         ImmutableMap.of(
-            DATA_SOURCE,
+            TestDataSource.WIKI,
             ImmutableList.of(
                 Intervals.of("2017-11-16T04:00:00/2017-11-16T10:00:00"),
                 Intervals.of("2017-11-16T14:00:00/2017-11-16T20:00:00")
             )
-        )
+        ),
+        statusTracker
     );
 
     assertCompactSegmentIntervals(
@@ -582,14 +587,15 @@ public class NewestSegmentFirstPolicyTest
   public void testIteratorReturnsSegmentsInConfiguredSegmentGranularity()
   {
     final SegmentTimeline timeline = createTimeline(
-        // Segments with day interval from Oct to Dec
-        new SegmentGenerateSpec(Intervals.of("2017-10-01T00:00:00/2017-12-31T00:00:00"), new Period("P1D"))
+        createSegments()
+            .forIntervals(3, Granularities.MONTH)
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.MONTH, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.MONTH),
+        timeline
     );
 
     // We should get all segments in timeline back since skip offset is P0D.
@@ -630,15 +636,14 @@ public class NewestSegmentFirstPolicyTest
   public void testIteratorReturnsSegmentsInMultipleIntervalIfConfiguredSegmentGranularityCrossBoundary()
   {
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(Intervals.of("2020-01-01/2020-01-08"), new Period("P7D")),
-        new SegmentGenerateSpec(Intervals.of("2020-01-28/2020-02-03"), new Period("P7D")),
-        new SegmentGenerateSpec(Intervals.of("2020-02-08/2020-02-15"), new Period("P7D"))
+        createSegments().forIntervals(1, Granularities.WEEK).startingAt("2020-01-01").withNumPartitions(4),
+        createSegments().forIntervals(1, Granularities.WEEK).startingAt("2020-01-28").withNumPartitions(4),
+        createSegments().forIntervals(1, Granularities.WEEK).startingAt("2020-02-08").withNumPartitions(4)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.MONTH, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.MONTH),
+        timeline
     );
     // We should get the segment of "2020-01-28/2020-02-03" back twice when the iterator returns for Jan and when the
     // iterator returns for Feb.
@@ -667,13 +672,12 @@ public class NewestSegmentFirstPolicyTest
   public void testIteratorDoesNotReturnCompactedInterval()
   {
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(Intervals.of("2017-12-01T00:00:00/2017-12-02T00:00:00"), new Period("P1D"))
+        createSegments().forIntervals(1, Granularities.DAY).startingAt("2017-12-01").withNumPartitions(4)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(40000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.MINUTE, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.MINUTE),
+        timeline
     );
 
     final List<DataSegment> expectedSegmentsToCompact = new ArrayList<>(
@@ -692,14 +696,15 @@ public class NewestSegmentFirstPolicyTest
   public void testIteratorReturnsAllMixedVersionSegmentsInConfiguredSegmentGranularity()
   {
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"), new Period("P1D"), "1994-04-29T00:00:00.000Z", null),
-        new SegmentGenerateSpec(Intervals.of("2017-10-01T01:00:00/2017-10-01T02:00:00"), new Period("PT1H"), "1994-04-30T00:00:00.000Z", null)
+        createSegments().forIntervals(1, Granularities.DAY).startingAt("2017-10-01")
+                        .withVersion("v1").withNumPartitions(4),
+        createSegments().forIntervals(1, Granularities.HOUR).startingAt("2017-10-01")
+                        .withVersion("v2").withNumPartitions(4)
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.MONTH, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.MONTH),
+        timeline
     );
 
     // We should get all segments in timeline back since skip offset is P0D.
@@ -725,25 +730,20 @@ public class NewestSegmentFirstPolicyTest
 
     // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        )
+        createSegments()
+            .startingAt("2017-10-01")
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, null))
+            .withNumPartitions(4),
+        createSegments()
+            .startingAt("2017-10-02")
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, null))
+            .withNumPartitions(4)
     );
 
     // Auto compaction config sets segmentGranularity=DAY
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.DAY),
+        timeline
     );
     Assert.assertFalse(iterator.hasNext());
   }
@@ -757,26 +757,20 @@ public class NewestSegmentFirstPolicyTest
     PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null));
 
     // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY
+    final CompactionState compactionState
+        = new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("segmentGranularity", "day"));
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("segmentGranularity", "day"))
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("segmentGranularity", "day"))
-        )
+        createSegments()
+            .forIntervals(2, Granularities.DAY)
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(compactionState)
     );
 
     // Auto compaction config sets segmentGranularity=DAY
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.DAY),
+        timeline
     );
     Assert.assertFalse(iterator.hasNext());
   }
@@ -790,26 +784,19 @@ public class NewestSegmentFirstPolicyTest
     PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null));
 
     // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY
+    final CompactionState compactionState = new CompactionState(partitionsSpec, null, null, null, indexSpec, null);
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        )
+        createSegments()
+            .forIntervals(2, Granularities.DAY)
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(compactionState)
     );
 
     // Auto compaction config sets segmentGranularity=YEAR
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.YEAR),
+        timeline
     );
     // We should get all segments in timeline back since skip offset is P0D.
     Assert.assertTrue(iterator.hasNext());
@@ -833,26 +820,20 @@ public class NewestSegmentFirstPolicyTest
     PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null));
 
     // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY
+    final CompactionState compactionState
+        = new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("segmentGranularity", "day"));
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("segmentGranularity", "day"))
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("segmentGranularity", "day"))
-        )
+        createSegments()
+            .forIntervals(2, Granularities.DAY)
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(compactionState)
     );
 
     // Auto compaction config sets segmentGranularity=YEAR
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.YEAR),
+        timeline
     );
     // We should get all segments in timeline back since skip offset is P0D.
     Assert.assertTrue(iterator.hasNext());
@@ -877,34 +858,28 @@ public class NewestSegmentFirstPolicyTest
 
     // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        )
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-02")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, null))
     );
 
     // Duration of new segmentGranularity is the same as before (P1D),
     // but we changed the timezone from UTC to Bangkok in the auto compaction spec
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(
-                            130000,
-                            new Period("P0D"),
-                            new UserCompactionTaskGranularityConfig(
-                                new PeriodGranularity(
-                                    new Period("P1D"),
-                                    null,
-                                    DateTimeZone.forTimeZone(TimeZone.getTimeZone("Asia/Bangkok"))
-                                ),
-                                null,
-                                null
-                            )
-                        )
-        ),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(
+                new PeriodGranularity(
+                    Period.days(1),
+                    null,
+                    DateTimeZone.forTimeZone(TimeZone.getTimeZone("Asia/Bangkok"))
+                ),
+                null,
+                null
+            )
+        ).build(),
+        timeline
     );
     // We should get all segments in timeline back since skip offset is P0D.
     Assert.assertTrue(iterator.hasNext());
@@ -929,33 +904,27 @@ public class NewestSegmentFirstPolicyTest
 
     // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        )
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-02")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, null))
     );
 
     // Duration of new segmentGranularity is the same as before (P1D), but we changed the origin in the autocompaction spec
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(
-                            130000,
-                            new Period("P0D"),
-                            new UserCompactionTaskGranularityConfig(
-                                new PeriodGranularity(
-                                    new Period("P1D"),
-                                    DateTimes.of("2012-01-02T00:05:00.000Z"),
-                                    DateTimeZone.UTC
-                                ),
-                                null,
-                                null
-                            )
-                        )
-        ),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(
+                new PeriodGranularity(
+                    Period.days(1),
+                    DateTimes.of("2012-01-02T00:05:00.000Z"),
+                    DateTimeZone.UTC
+                ),
+                null,
+                null
+            )
+        ).build(),
+        timeline
     );
     // We should get all segments in timeline back since skip offset is P0D.
     Assert.assertTrue(iterator.hasNext());
@@ -983,31 +952,29 @@ public class NewestSegmentFirstPolicyTest
     // rollup=true for interval 2017-10-02T00:00:00/2017-10-03T00:00:00,
     // and rollup=null for interval 2017-10-03T00:00:00/2017-10-04T00:00:00 (queryGranularity was not set during last compaction)
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("rollup", "false"))
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("rollup", "true"))
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-03T00:00:00/2017-10-04T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of())
-        )
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("rollup", "false"))),
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-02")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("rollup", "true"))),
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-03")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of()))
     );
 
     // Auto compaction config sets rollup=true
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(null, null, true))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(null, null, true)
+        ).build(),
+        timeline
     );
     // We should get interval 2017-10-01T00:00:00/2017-10-02T00:00:00 and interval 2017-10-03T00:00:00/2017-10-04T00:00:00.
     Assert.assertTrue(iterator.hasNext());
@@ -1043,31 +1010,29 @@ public class NewestSegmentFirstPolicyTest
     // queryGranularity=MINUTE for interval 2017-10-02T00:00:00/2017-10-03T00:00:00,
     // and queryGranularity=null for interval 2017-10-03T00:00:00/2017-10-04T00:00:00 (queryGranularity was not set during last compaction)
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("queryGranularity", "day"))
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("queryGranularity", "minute"))
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-03T00:00:00/2017-10-04T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of())
-        )
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("queryGranularity", "day"))),
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-02")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of("queryGranularity", "minute"))),
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-03")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, ImmutableMap.of()))
     );
 
     // Auto compaction config sets queryGranularity=MINUTE
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(null, Granularities.MINUTE, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(null, Granularities.MINUTE, null)
+        ).build(),
+        timeline
     );
     // We should get interval 2017-10-01T00:00:00/2017-10-02T00:00:00 and interval 2017-10-03T00:00:00/2017-10-04T00:00:00.
     Assert.assertTrue(iterator.hasNext());
@@ -1104,44 +1069,34 @@ public class NewestSegmentFirstPolicyTest
     // Dimensions=null for interval 2017-10-03T00:00:00/2017-10-04T00:00:00 (dimensions was not set during last compaction)
     // and dimensionsSpec=null for interval 2017-10-04T00:00:00/2017-10-05T00:00:00 (dimensionsSpec was not set during last compaction)
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo"))), null, null, indexSpec, null)
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo"))), null, null, indexSpec, null)
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-03T00:00:00/2017-10-04T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, DimensionsSpec.EMPTY, null, null, indexSpec, null)
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-04T00:00:00/2017-10-05T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        )
+        createSegments()
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(partitionsSpec, new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo"))), null, null, indexSpec, null)
+            ),
+        createSegments()
+            .startingAt("2017-10-02")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(partitionsSpec, new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo"))), null, null, indexSpec, null)
+            ),
+        createSegments()
+            .startingAt("2017-10-03")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, DimensionsSpec.EMPTY, null, null, indexSpec, null)),
+        createSegments()
+            .startingAt("2017-10-04")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, null))
     );
 
     // Auto compaction config sets Dimensions=["foo"]
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(
-            130000,
-            new Period("P0D"),
-            null,
-            new UserCompactionTaskDimensionsConfig(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo"))),
-            null,
-            null
-        )),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withDimensionsSpec(
+            new UserCompactionTaskDimensionsConfig(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo")))
+        ).build(),
+        timeline
     );
     // We should get interval 2017-10-01T00:00:00/2017-10-02T00:00:00, interval 2017-10-04T00:00:00/2017-10-05T00:00:00, and interval 2017-10-03T00:00:00/2017-10-04T00:00:00.
     Assert.assertTrue(iterator.hasNext());
@@ -1172,17 +1127,11 @@ public class NewestSegmentFirstPolicyTest
     Assert.assertFalse(iterator.hasNext());
 
     // Auto compaction config sets Dimensions=null
-    iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(
-            130000,
-            new Period("P0D"),
-            null,
-            new UserCompactionTaskDimensionsConfig(null),
-            null,
-            null
-        )),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    iterator = createIterator(
+        configBuilder().withDimensionsSpec(
+            new UserCompactionTaskDimensionsConfig(null)
+        ).build(),
+        timeline
     );
     // No more
     Assert.assertFalse(iterator.hasNext());
@@ -1203,65 +1152,57 @@ public class NewestSegmentFirstPolicyTest
     // filter=null for interval 2017-10-03T00:00:00/2017-10-04T00:00:00 (filter was not set during last compaction)
     // and transformSpec=null for interval 2017-10-04T00:00:00/2017-10-05T00:00:00 (transformSpec was not set during last compaction)
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(
-                partitionsSpec,
-                null,
-                null,
-                mapper.readValue(mapper.writeValueAsString(new TransformSpec(new SelectorDimFilter("dim1", "foo", null), null)), new TypeReference<Map<String, Object>>() {}),
-                indexSpec,
-                null
-            )
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(
-                partitionsSpec,
-                null,
-                null,
-                mapper.readValue(mapper.writeValueAsString(new TransformSpec(new SelectorDimFilter("dim1", "bar", null), null)), new TypeReference<Map<String, Object>>() {}),
-                indexSpec,
-                null
-            )
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-03T00:00:00/2017-10-04T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(
-                partitionsSpec,
-                null,
-                null,
-                mapper.readValue(mapper.writeValueAsString(new TransformSpec(null, null)), new TypeReference<Map<String, Object>>() {}),
-                indexSpec,
-                null
-            )
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-04T00:00:00/2017-10-05T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        )
+        createSegments()
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(
+                    partitionsSpec,
+                    null,
+                    null,
+                    mapper.readValue(mapper.writeValueAsString(new TransformSpec(new SelectorDimFilter("dim1", "foo", null), null)), new TypeReference<Map<String, Object>>() {}),
+                    indexSpec,
+                    null
+                )
+            ),
+        createSegments()
+            .startingAt("2017-10-02")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(
+                    partitionsSpec,
+                    null,
+                    null,
+                    mapper.readValue(mapper.writeValueAsString(new TransformSpec(new SelectorDimFilter("dim1", "bar", null), null)), new TypeReference<Map<String, Object>>() {}),
+                    indexSpec,
+                    null
+                )
+            ),
+        createSegments()
+            .startingAt("2017-10-03")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(
+                    partitionsSpec,
+                    null,
+                    null,
+                    mapper.readValue(mapper.writeValueAsString(new TransformSpec(null, null)), new TypeReference<Map<String, Object>>() {}),
+                    indexSpec,
+                    null
+                )
+            ),
+        createSegments()
+            .startingAt("2017-10-04")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, null))
     );
 
     // Auto compaction config sets filter=SelectorDimFilter("dim1", "bar", null)
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(
-            130000,
-            new Period("P0D"),
-            null,
-            null,
-            new UserCompactionTaskTransformConfig(new SelectorDimFilter("dim1", "bar", null)),
-            null
-        )),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withTransformSpec(
+            new UserCompactionTaskTransformConfig(new SelectorDimFilter("dim1", "bar", null))
+        ).build(),
+        timeline
     );
     // We should get interval 2017-10-01T00:00:00/2017-10-02T00:00:00, interval 2017-10-04T00:00:00/2017-10-05T00:00:00, and interval 2017-10-03T00:00:00/2017-10-04T00:00:00.
     Assert.assertTrue(iterator.hasNext());
@@ -1292,17 +1233,11 @@ public class NewestSegmentFirstPolicyTest
     Assert.assertFalse(iterator.hasNext());
 
     // Auto compaction config sets filter=null
-    iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(
-            130000,
-            new Period("P0D"),
-            null,
-            null,
-            new UserCompactionTaskTransformConfig(null),
-            null
-        )),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    iterator = createIterator(
+        configBuilder().withTransformSpec(
+            new UserCompactionTaskTransformConfig(null)
+        ).build(),
+        timeline
     );
     // No more
     Assert.assertFalse(iterator.hasNext());
@@ -1327,65 +1262,57 @@ public class NewestSegmentFirstPolicyTest
     // metricsSpec=[] for interval 2017-10-03T00:00:00/2017-10-04T00:00:00 (filter was not set during last compaction)
     // and metricsSpec=null for interval 2017-10-04T00:00:00/2017-10-05T00:00:00 (transformSpec was not set during last compaction)
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(
-                partitionsSpec,
-                null,
-                mapper.convertValue(new AggregatorFactory[] {new CountAggregatorFactory("cnt")}, new TypeReference<List<Object>>() {}),
-                null,
-                indexSpec,
-                null
-            )
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(
-                partitionsSpec,
-                null,
-                mapper.convertValue(new AggregatorFactory[] {new CountAggregatorFactory("cnt"), new LongSumAggregatorFactory("val", "val")}, new TypeReference<List<Object>>() {}),
-                null,
-                indexSpec,
-                null
-            )
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-03T00:00:00/2017-10-04T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(
-                partitionsSpec,
-                null,
-                mapper.convertValue(new AggregatorFactory[] {}, new TypeReference<List<Object>>() {}),
-                null,
-                indexSpec,
-                null
-            )
-        ),
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-04T00:00:00/2017-10-05T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, indexSpec, null)
-        )
+        createSegments()
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(
+                    partitionsSpec,
+                    null,
+                    mapper.convertValue(new AggregatorFactory[] {new CountAggregatorFactory("cnt")}, new TypeReference<List<Object>>() {}),
+                    null,
+                    indexSpec,
+                    null
+                )
+            ),
+        createSegments()
+            .startingAt("2017-10-02")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(
+                    partitionsSpec,
+                    null,
+                    mapper.convertValue(new AggregatorFactory[] {new CountAggregatorFactory("cnt"), new LongSumAggregatorFactory("val", "val")}, new TypeReference<List<Object>>() {}),
+                    null,
+                    indexSpec,
+                    null
+                )
+            ),
+        createSegments()
+            .startingAt("2017-10-03")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(
+                    partitionsSpec,
+                    null,
+                    mapper.convertValue(new AggregatorFactory[] {}, new TypeReference<List<Object>>() {}),
+                    null,
+                    indexSpec,
+                    null
+                )
+            ),
+        createSegments()
+            .startingAt("2017-10-04")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, indexSpec, null))
     );
 
     // Auto compaction config sets metricsSpec={CountAggregatorFactory("cnt"), LongSumAggregatorFactory("val", "val")}
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(
-            130000,
-            new Period("P0D"),
-            null,
-            null,
-            null,
-            new AggregatorFactory[] {new CountAggregatorFactory("cnt"), new LongSumAggregatorFactory("val", "val")}
-        )),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withMetricsSpec(
+            new AggregatorFactory[]{new CountAggregatorFactory("cnt"), new LongSumAggregatorFactory("val", "val")}
+        ).build(),
+        timeline
     );
     // We should get interval 2017-10-01T00:00:00/2017-10-02T00:00:00, interval 2017-10-04T00:00:00/2017-10-05T00:00:00, and interval 2017-10-03T00:00:00/2017-10-04T00:00:00.
     Assert.assertTrue(iterator.hasNext());
@@ -1416,17 +1343,9 @@ public class NewestSegmentFirstPolicyTest
     Assert.assertFalse(iterator.hasNext());
 
     // Auto compaction config sets metricsSpec=null
-    iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(
-            130000,
-            new Period("P0D"),
-            null,
-            null,
-            null,
-            null
-        )),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    iterator = createIterator(
+        configBuilder().build(),
+        timeline
     );
     // No more
     Assert.assertFalse(iterator.hasNext());
@@ -1436,14 +1355,15 @@ public class NewestSegmentFirstPolicyTest
   public void testIteratorReturnsSegmentsSmallerSegmentGranularityCoveringMultipleSegmentsInTimeline()
   {
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"), new Period("P1D"), "1994-04-29T00:00:00.000Z", null),
-        new SegmentGenerateSpec(Intervals.of("2017-10-01T01:00:00/2017-10-01T02:00:00"), new Period("PT1H"), "1994-04-30T00:00:00.000Z", null)
+        createSegments().forIntervals(1, Granularities.DAY).startingAt("2017-10-01")
+                        .withNumPartitions(4).withVersion("v1"),
+        createSegments().forIntervals(1, Granularities.HOUR).startingAt("2017-10-01")
+                        .withNumPartitions(4).withVersion("v2")
     );
 
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null))),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        createConfigWithSegmentGranularity(Granularities.HOUR),
+        timeline
     );
 
     // We should get all segments in timeline back since skip offset is P0D.
@@ -1472,33 +1392,27 @@ public class NewestSegmentFirstPolicyTest
 
     // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(partitionsSpec, null, null, null, newIndexSpecMap, null)
-        )
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-02")
+            .withNumPartitions(4)
+            .withCompactionState(new CompactionState(partitionsSpec, null, null, null, newIndexSpecMap, null))
     );
 
     // Duration of new segmentGranularity is the same as before (P1D)
-    final CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(
-                            130000,
-                            new Period("P0D"),
-                            new UserCompactionTaskGranularityConfig(
-                                new PeriodGranularity(
-                                    new Period("P1D"),
-                                    null,
-                                    DateTimeZone.UTC
-                                ),
-                                null,
-                                null
-                            )
-                        )
-        ),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+    final CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(
+                new PeriodGranularity(
+                    Period.days(1),
+                    null,
+                    DateTimeZone.UTC
+                ),
+                null,
+                null
+            )
+        ).build(),
+        timeline
     );
     // We should get all segments in timeline back since indexSpec changed
     Assert.assertTrue(iterator.hasNext());
@@ -1519,28 +1433,17 @@ public class NewestSegmentFirstPolicyTest
     NullHandling.initializeForTests();
     PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null));
     final SegmentTimeline timeline = createTimeline(
-        new SegmentGenerateSpec(
-            Intervals.of("2017-10-01T00:00:00/2017-10-02T00:00:00"),
-            new Period("P1D"),
-            null,
-            new CompactionState(
-                partitionsSpec,
-                null,
-                null,
-                null,
-                IndexSpec.DEFAULT.asMap(mapper),
-                null
+        createSegments()
+            .forIntervals(1, Granularities.DAY)
+            .startingAt("2017-10-01")
+            .withNumPartitions(4)
+            .withCompactionState(
+                new CompactionState(partitionsSpec, null, null, null, IndexSpec.DEFAULT.asMap(mapper), null)
             )
-        )
     );
 
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(
-            130000,
-            new Period("P0D"),
-            null,
-            null,
-            null,
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withTuningConfig(
             new UserCompactionTaskQueryTuningConfig(
                 null,
                 new OnheapIncrementalIndex.Spec(true),
@@ -1561,21 +1464,14 @@ public class NewestSegmentFirstPolicyTest
                 null,
                 null,
                 null
-            ),
-            null
-        )),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+            )
+        ).build(),
+        timeline
     );
     Assert.assertFalse(iterator.hasNext());
 
-    iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE, createCompactionConfig(
-            130000,
-            new Period("P0D"),
-            null,
-            null,
-            null,
+    iterator = createIterator(
+        configBuilder().withTuningConfig(
             new UserCompactionTaskQueryTuningConfig(
                 null,
                 new OnheapIncrementalIndex.Spec(false),
@@ -1596,11 +1492,9 @@ public class NewestSegmentFirstPolicyTest
                 null,
                 null,
                 null
-            ),
-            null
-        )),
-        ImmutableMap.of(DATA_SOURCE, timeline),
-        Collections.emptyMap()
+            )
+        ).build(),
+        timeline
     );
     Assert.assertFalse(iterator.hasNext());
   }
@@ -1608,30 +1502,22 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testSkipAllGranularityToDefault()
   {
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(10000,
-                                               new Period("P0D"),
-                                               null
-                        )
-        ),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            SegmentTimeline.forSegments(ImmutableSet.of(
-                                            new DataSegment(
-                                                DATA_SOURCE,
-                                                Intervals.ETERNITY,
-                                                "0",
-                                                new HashMap<>(),
-                                                new ArrayList<>(),
-                                                new ArrayList<>(),
-                                                new NumberedShardSpec(0, 0),
-                                                0,
-                                                100)
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().build(),
+        SegmentTimeline.forSegments(ImmutableSet.of(
+                                        new DataSegment(
+                                            TestDataSource.WIKI,
+                                            Intervals.ETERNITY,
+                                            "0",
+                                            new HashMap<>(),
+                                            new ArrayList<>(),
+                                            new ArrayList<>(),
+                                            new NumberedShardSpec(0, 0),
+                                            0,
+                                            100
                                         )
-            )
-        ),
-        Collections.emptyMap()
+                                    )
+        )
     );
 
     Assert.assertFalse(iterator.hasNext());
@@ -1640,30 +1526,22 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testSkipFirstHalfEternityToDefault()
   {
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(10000,
-                                               new Period("P0D"),
-                                               null
-                        )
-        ),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            SegmentTimeline.forSegments(ImmutableSet.of(
-                                            new DataSegment(
-                                                DATA_SOURCE,
-                                                new Interval(DateTimes.MIN, DateTimes.of("2024-01-01")),
-                                                "0",
-                                                new HashMap<>(),
-                                                new ArrayList<>(),
-                                                new ArrayList<>(),
-                                                new NumberedShardSpec(0, 0),
-                                                0,
-                                                100)
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().build(),
+        SegmentTimeline.forSegments(ImmutableSet.of(
+                                        new DataSegment(
+                                            TestDataSource.WIKI,
+                                            new Interval(DateTimes.MIN, DateTimes.of("2024-01-01")),
+                                            "0",
+                                            new HashMap<>(),
+                                            new ArrayList<>(),
+                                            new ArrayList<>(),
+                                            new NumberedShardSpec(0, 0),
+                                            0,
+                                            100
                                         )
-            )
-        ),
-        Collections.emptyMap()
+                                    )
+        )
     );
 
     Assert.assertFalse(iterator.hasNext());
@@ -1672,30 +1550,22 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testSkipSecondHalfOfEternityToDefault()
   {
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(10000,
-                                               new Period("P0D"),
-                                               null
-                        )
-        ),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            SegmentTimeline.forSegments(ImmutableSet.of(
-                                            new DataSegment(
-                                                DATA_SOURCE,
-                                                new Interval(DateTimes.of("2024-01-01"), DateTimes.MAX),
-                                                "0",
-                                                new HashMap<>(),
-                                                new ArrayList<>(),
-                                                new ArrayList<>(),
-                                                new NumberedShardSpec(0, 0),
-                                                0,
-                                                100)
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().build(),
+        SegmentTimeline.forSegments(ImmutableSet.of(
+                                        new DataSegment(
+                                            TestDataSource.WIKI,
+                                            new Interval(DateTimes.of("2024-01-01"), DateTimes.MAX),
+                                            "0",
+                                            new HashMap<>(),
+                                            new ArrayList<>(),
+                                            new ArrayList<>(),
+                                            new NumberedShardSpec(0, 0),
+                                            0,
+                                            100
                                         )
-            )
-        ),
-        Collections.emptyMap()
+                                    )
+        )
     );
 
     Assert.assertFalse(iterator.hasNext());
@@ -1704,30 +1574,24 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testSkipAllToAllGranularity()
   {
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(10000,
-                                               new Period("P0D"),
-                                               new UserCompactionTaskGranularityConfig(Granularities.ALL, null, null)
-                        )
-        ),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            SegmentTimeline.forSegments(ImmutableSet.of(
-                                            new DataSegment(
-                                                DATA_SOURCE,
-                                                Intervals.ETERNITY,
-                                                "0",
-                                                new HashMap<>(),
-                                                new ArrayList<>(),
-                                                new ArrayList<>(),
-                                                new NumberedShardSpec(0, 0),
-                                                0,
-                                                100)
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(Granularities.ALL, null, null)
+        ).build(),
+        SegmentTimeline.forSegments(ImmutableSet.of(
+                                        new DataSegment(
+                                            TestDataSource.WIKI,
+                                            Intervals.ETERNITY,
+                                            "0",
+                                            new HashMap<>(),
+                                            new ArrayList<>(),
+                                            new ArrayList<>(),
+                                            new NumberedShardSpec(0, 0),
+                                            0,
+                                            100
                                         )
-            )
-        ),
-        Collections.emptyMap()
+                                    )
+        )
     );
 
     Assert.assertFalse(iterator.hasNext());
@@ -1736,30 +1600,24 @@ public class NewestSegmentFirstPolicyTest
   @Test
   public void testSkipAllToFinerGranularity()
   {
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(10000,
-                                               new Period("P0D"),
-                                               new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)
-                        )
-        ),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            SegmentTimeline.forSegments(ImmutableSet.of(
-                                            new DataSegment(
-                                                DATA_SOURCE,
-                                                Intervals.ETERNITY,
-                                                "0",
-                                                new HashMap<>(),
-                                                new ArrayList<>(),
-                                                new ArrayList<>(),
-                                                new NumberedShardSpec(0, 0),
-                                                0,
-                                                100)
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)
+        ).build(),
+        SegmentTimeline.forSegments(ImmutableSet.of(
+                                        new DataSegment(
+                                            TestDataSource.WIKI,
+                                            Intervals.ETERNITY,
+                                            "0",
+                                            new HashMap<>(),
+                                            new ArrayList<>(),
+                                            new ArrayList<>(),
+                                            new NumberedShardSpec(0, 0),
+                                            0,
+                                            100
                                         )
-            )
-        ),
-        Collections.emptyMap()
+                                    )
+        )
     );
 
     Assert.assertFalse(iterator.hasNext());
@@ -1769,7 +1627,7 @@ public class NewestSegmentFirstPolicyTest
   public void testSkipCompactionForIntervalsContainingSingleTombstone()
   {
     final DataSegment tombstone2023 = new DataSegment(
-        DATA_SOURCE,
+        TestDataSource.WIKI,
         Intervals.of("2023/2024"),
         "0",
         new HashMap<>(),
@@ -1779,7 +1637,7 @@ public class NewestSegmentFirstPolicyTest
         0,
         1);
     final DataSegment dataSegment2023 = new DataSegment(
-        DATA_SOURCE,
+        TestDataSource.WIKI,
         Intervals.of("2023/2024"),
         "0",
         new HashMap<>(),
@@ -1789,7 +1647,7 @@ public class NewestSegmentFirstPolicyTest
         0,
         100);
     final DataSegment tombstone2024 = new DataSegment(
-        DATA_SOURCE,
+        TestDataSource.WIKI,
         Intervals.of("2024/2025"),
         "0",
         new HashMap<>(),
@@ -1799,18 +1657,11 @@ public class NewestSegmentFirstPolicyTest
         0,
         1);
 
-    CompactionSegmentIterator iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(10000,
-                                               new Period("P0D"),
-                                               new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null)
-                        )
-        ),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            SegmentTimeline.forSegments(ImmutableSet.of(tombstone2023, dataSegment2023, tombstone2024))
-        ),
-        Collections.emptyMap()
+    CompactionSegmentIterator iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null)
+        ).build(),
+        SegmentTimeline.forSegments(ImmutableSet.of(tombstone2023, dataSegment2023, tombstone2024))
     );
 
     // Skips 2024/2025 since it has a single tombstone and no data.
@@ -1821,7 +1672,7 @@ public class NewestSegmentFirstPolicyTest
     );
 
     final DataSegment tombstone2025Jan = new DataSegment(
-        DATA_SOURCE,
+        TestDataSource.WIKI,
         Intervals.of("2025-01-01/2025-02-01"),
         "0",
         new HashMap<>(),
@@ -1831,7 +1682,7 @@ public class NewestSegmentFirstPolicyTest
         0,
         1);
     final DataSegment tombstone2025Feb = new DataSegment(
-        DATA_SOURCE,
+        TestDataSource.WIKI,
         Intervals.of("2025-02-01/2025-03-01"),
         "0",
         new HashMap<>(),
@@ -1841,7 +1692,7 @@ public class NewestSegmentFirstPolicyTest
         0,
         1);
     final DataSegment tombstone2025Mar = new DataSegment(
-        DATA_SOURCE,
+        TestDataSource.WIKI,
         Intervals.of("2025-03-01/2025-04-01"),
         "0",
         new HashMap<>(),
@@ -1850,25 +1701,18 @@ public class NewestSegmentFirstPolicyTest
         TombstoneShardSpec.INSTANCE,
         0,
         1);
-    iterator = policy.createIterator(
-        ImmutableMap.of(DATA_SOURCE,
-                        createCompactionConfig(10000,
-                                               new Period("P0D"),
-                                               new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null)
-                        )
-        ),
-        ImmutableMap.of(
-            DATA_SOURCE,
-            SegmentTimeline.forSegments(ImmutableSet.of(
-                tombstone2023,
-                dataSegment2023,
-                tombstone2024,
-                tombstone2025Jan,
-                tombstone2025Feb,
-                tombstone2025Mar
-            ))
-        ),
-        Collections.emptyMap()
+    iterator = createIterator(
+        configBuilder().withGranularitySpec(
+            new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null)
+        ).build(),
+        SegmentTimeline.forSegments(ImmutableSet.of(
+            tombstone2023,
+            dataSegment2023,
+            tombstone2024,
+            tombstone2025Jan,
+            tombstone2025Feb,
+            tombstone2025Mar
+        ))
     );
     // Does not skip the tombstones in 2025 since there are multiple of them which could potentially be condensed to one
     Assert.assertEquals(
@@ -1877,6 +1721,61 @@ public class NewestSegmentFirstPolicyTest
     );
   }
 
+  @Test
+  public void testPriorityDatasource()
+  {
+    final List<DataSegment> wikiSegments
+        = CreateDataSegments.ofDatasource(TestDataSource.WIKI)
+                            .forIntervals(1, Granularities.DAY)
+                            .startingAt("2012-01-01")
+                            .withNumPartitions(10)
+                            .eachOfSizeInMb(100);
+    final List<DataSegment> koalaSegments
+        = CreateDataSegments.ofDatasource(TestDataSource.KOALA)
+                            .forIntervals(1, Granularities.DAY)
+                            .startingAt("2013-01-01")
+                            .withNumPartitions(10)
+                            .eachOfSizeInMb(100);
+
+    // Setup policy and iterator with priorityDatasource = WIKI
+    final NewestSegmentFirstPolicy policy = new NewestSegmentFirstPolicy(TestDataSource.WIKI);
+    CompactionSegmentIterator iterator = new PriorityBasedCompactionSegmentIterator(
+        policy,
+        ImmutableMap.of(
+            TestDataSource.WIKI, configBuilder().forDataSource(TestDataSource.WIKI).build(),
+            TestDataSource.KOALA, configBuilder().forDataSource(TestDataSource.KOALA).build()
+        ),
+        ImmutableMap.of(
+            TestDataSource.WIKI, SegmentTimeline.forSegments(wikiSegments),
+            TestDataSource.KOALA, SegmentTimeline.forSegments(koalaSegments)
+        ),
+        Collections.emptyMap(),
+        statusTracker
+    );
+
+    // Verify that the segments of WIKI are preferred even though they are older
+    Assert.assertTrue(iterator.hasNext());
+    CompactionCandidate next = iterator.next();
+    Assert.assertEquals(TestDataSource.WIKI, next.getDataSource());
+    Assert.assertEquals(Intervals.of("2012-01-01/P1D"), next.getUmbrellaInterval());
+
+    Assert.assertTrue(iterator.hasNext());
+    next = iterator.next();
+    Assert.assertEquals(TestDataSource.KOALA, next.getDataSource());
+    Assert.assertEquals(Intervals.of("2013-01-01/P1D"), next.getUmbrellaInterval());
+  }
+
+  private CompactionSegmentIterator createIterator(DataSourceCompactionConfig config, SegmentTimeline timeline)
+  {
+    return new PriorityBasedCompactionSegmentIterator(
+        policy,
+        Collections.singletonMap(TestDataSource.WIKI, config),
+        Collections.singletonMap(TestDataSource.WIKI, timeline),
+        Collections.emptyMap(),
+        statusTracker
+    );
+  }
+
   private static void assertCompactSegmentIntervals(
       CompactionSegmentIterator iterator,
       Period segmentPeriod,
@@ -1922,145 +1821,36 @@ public class NewestSegmentFirstPolicyTest
     }
   }
 
-  private static SegmentTimeline createTimeline(SegmentGenerateSpec... specs)
+  private static CreateDataSegments createSegments()
   {
-    List<DataSegment> segments = new ArrayList<>();
-    final String version = DateTimes.nowUtc().toString();
-
-    final List<SegmentGenerateSpec> orderedSpecs = Arrays.asList(specs);
-    orderedSpecs.sort(Comparator.comparing(s -> s.totalInterval, Comparators.intervalsByStartThenEnd().reversed()));
-
-    for (SegmentGenerateSpec spec : orderedSpecs) {
-      Interval remainingInterval = spec.totalInterval;
-
-      while (!Intervals.isEmpty(remainingInterval)) {
-        final Interval segmentInterval;
-        if (remainingInterval.toDuration().isLongerThan(spec.segmentPeriod.toStandardDuration())) {
-          segmentInterval = new Interval(spec.segmentPeriod, remainingInterval.getEnd());
-        } else {
-          segmentInterval = remainingInterval;
-        }
-
-        for (int i = 0; i < spec.numSegmentsPerShard; i++) {
-          final ShardSpec shardSpec = new NumberedShardSpec(i, spec.numSegmentsPerShard);
-          final DataSegment segment = new DataSegment(
-              DATA_SOURCE,
-              segmentInterval,
-              spec.version == null ? version : spec.version,
-              null,
-              ImmutableList.of(),
-              ImmutableList.of(),
-              shardSpec,
-              spec.lastCompactionState,
-              0,
-              spec.segmentSize
-          );
-          segments.add(segment);
-        }
-
-        remainingInterval = removeIntervalFromEnd(remainingInterval, segmentInterval);
-      }
-    }
-
-    return SegmentTimeline.forSegments(segments);
+    return CreateDataSegments.ofDatasource(TestDataSource.WIKI);
   }
 
-  /**
-   * Returns an interval [largeInterval.start - smallInterval.start) given that
-   * the end of both intervals is the same.
-   */
-  private static Interval removeIntervalFromEnd(Interval largeInterval, Interval smallInterval)
-  {
-    Preconditions.checkArgument(
-        largeInterval.getEnd().equals(smallInterval.getEnd()),
-        "end should be same. largeInterval[%s] smallInterval[%s]",
-        largeInterval,
-        smallInterval
-    );
-    return new Interval(largeInterval.getStart(), smallInterval.getStart());
-  }
-
-  private DataSourceCompactionConfig createCompactionConfig(
-      long inputSegmentSizeBytes,
-      Period skipOffsetFromLatest,
-      UserCompactionTaskGranularityConfig granularitySpec
+  private static SegmentTimeline createTimeline(
+      CreateDataSegments... segmentBuilders
   )
   {
-    return createCompactionConfig(inputSegmentSizeBytes, skipOffsetFromLatest, granularitySpec, null, null, null, null);
+    final SegmentTimeline timeline = new SegmentTimeline();
+    for (CreateDataSegments builder : segmentBuilders) {
+      timeline.addSegments(builder.eachOfSizeInMb(100).iterator());
+    }
+
+    return timeline;
   }
 
-  private DataSourceCompactionConfig createCompactionConfig(
-      long inputSegmentSizeBytes,
-      Period skipOffsetFromLatest,
-      UserCompactionTaskGranularityConfig granularitySpec,
-      UserCompactionTaskDimensionsConfig dimensionsSpec,
-      UserCompactionTaskTransformConfig transformSpec,
-      AggregatorFactory[] metricsSpec
+  private static DataSourceCompactionConfig createConfigWithSegmentGranularity(
+      Granularity segmentGranularity 
   )
   {
-    return createCompactionConfig(inputSegmentSizeBytes, skipOffsetFromLatest, granularitySpec, dimensionsSpec, transformSpec, null, metricsSpec);
+    return configBuilder().withGranularitySpec(
+        new UserCompactionTaskGranularityConfig(segmentGranularity, null, null)
+    ).build();
   }
 
-  private DataSourceCompactionConfig createCompactionConfig(
-      long inputSegmentSizeBytes,
-      Period skipOffsetFromLatest,
-      UserCompactionTaskGranularityConfig granularitySpec,
-      UserCompactionTaskDimensionsConfig dimensionsSpec,
-      UserCompactionTaskTransformConfig transformSpec,
-      UserCompactionTaskQueryTuningConfig tuningConfig,
-      AggregatorFactory[] metricsSpec
-  )
+  private static DataSourceCompactionConfig.Builder configBuilder()
   {
-    return new DataSourceCompactionConfig(
-        DATA_SOURCE,
-        0,
-        inputSegmentSizeBytes,
-        null,
-        skipOffsetFromLatest,
-        tuningConfig,
-        granularitySpec,
-        dimensionsSpec,
-        metricsSpec,
-        transformSpec,
-        null,
-        null,
-        null
-    );
-  }
-
-  private static class SegmentGenerateSpec
-  {
-    private final Interval totalInterval;
-    private final Period segmentPeriod;
-    private final long segmentSize;
-    private final int numSegmentsPerShard;
-    private final String version;
-    private final CompactionState lastCompactionState;
-
-    SegmentGenerateSpec(Interval totalInterval, Period segmentPeriod)
-    {
-      this(totalInterval, segmentPeriod, null, null);
-    }
-
-    SegmentGenerateSpec(Interval totalInterval, Period segmentPeriod, String version, CompactionState lastCompactionState)
-    {
-      this(totalInterval, segmentPeriod, DEFAULT_SEGMENT_SIZE, DEFAULT_NUM_SEGMENTS_PER_SHARD, version, lastCompactionState);
-    }
-
-    SegmentGenerateSpec(Interval totalInterval, Period segmentPeriod, long segmentSize, int numSegmentsPerShard)
-    {
-      this(totalInterval, segmentPeriod, segmentSize, numSegmentsPerShard, null, null);
-    }
-
-    SegmentGenerateSpec(Interval totalInterval, Period segmentPeriod, long segmentSize, int numSegmentsPerShard, String version, CompactionState lastCompactionState)
-    {
-      Preconditions.checkArgument(numSegmentsPerShard >= 1);
-      this.totalInterval = totalInterval;
-      this.segmentPeriod = segmentPeriod;
-      this.segmentSize = segmentSize;
-      this.numSegmentsPerShard = numSegmentsPerShard;
-      this.version = version;
-      this.lastCompactionState = lastCompactionState;
-    }
+    return DataSourceCompactionConfig.builder()
+                                     .forDataSource(TestDataSource.WIKI)
+                                     .withSkipOffsetFromLatest(Period.seconds(0));
   }
 }
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/AutoCompactionSnapshotTest.java b/server/src/test/java/org/apache/druid/server/coordinator/AutoCompactionSnapshotTest.java
index 5517bf9e6a4..9a8cd3cc877 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/AutoCompactionSnapshotTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/AutoCompactionSnapshotTest.java
@@ -19,7 +19,7 @@
 
 package org.apache.druid.server.coordinator;
 
-import org.apache.druid.server.coordinator.compact.CompactionStatistics;
+import org.apache.druid.server.compaction.CompactionStatistics;
 import org.junit.Assert;
 import org.junit.Test;
 
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CreateDataSegments.java b/server/src/test/java/org/apache/druid/server/coordinator/CreateDataSegments.java
index e3c31f6471c..b5e6879bedf 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/CreateDataSegments.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/CreateDataSegments.java
@@ -24,6 +24,7 @@ import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.segment.IndexIO;
+import org.apache.druid.timeline.CompactionState;
 import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.partition.NumberedShardSpec;
 import org.joda.time.DateTime;
@@ -50,6 +51,9 @@ public class CreateDataSegments
   private int numPartitions = 1;
   private int numIntervals = 1;
 
+  private String version = "1";
+  private CompactionState compactionState = null;
+
   public static CreateDataSegments ofDatasource(String datasource)
   {
     return new CreateDataSegments(datasource);
@@ -85,7 +89,24 @@ public class CreateDataSegments
     return this;
   }
 
+  public CreateDataSegments withCompactionState(CompactionState compactionState)
+  {
+    this.compactionState = compactionState;
+    return this;
+  }
+
+  public CreateDataSegments withVersion(String version)
+  {
+    this.version = version;
+    return this;
+  }
+
   public List<DataSegment> eachOfSizeInMb(long sizeMb)
+  {
+    return eachOfSize(sizeMb * 1_000_000);
+  }
+
+  public List<DataSegment> eachOfSize(long sizeInBytes)
   {
     boolean isEternityInterval = Objects.equals(granularity, Granularities.ALL);
     if (isEternityInterval) {
@@ -105,9 +126,11 @@ public class CreateDataSegments
             new NumberedDataSegment(
                 datasource,
                 nextInterval,
+                version,
                 new NumberedShardSpec(numPartition, numPartitions),
                 ++uniqueIdInInterval,
-                sizeMb << 20
+                compactionState,
+                sizeInBytes
             )
         );
       }
@@ -128,23 +151,26 @@ public class CreateDataSegments
     private NumberedDataSegment(
         String datasource,
         Interval interval,
+        String version,
         NumberedShardSpec shardSpec,
-        int uinqueId,
+        int uniqueId,
+        CompactionState compactionState,
         long size
     )
     {
       super(
           datasource,
           interval,
-          "1",
+          version,
           Collections.emptyMap(),
           Collections.emptyList(),
           Collections.emptyList(),
           shardSpec,
+          compactionState,
           IndexIO.CURRENT_VERSION_ID,
           size
       );
-      this.uniqueId = uinqueId;
+      this.uniqueId = uniqueId;
     }
 
     @Override
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigAuditEntryTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigAuditEntryTest.java
index 27d80e3b568..bdd02846921 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigAuditEntryTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigAuditEntryTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.server.coordinator;
 import org.apache.druid.audit.AuditInfo;
 import org.apache.druid.indexer.CompactionEngine;
 import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.segment.TestDataSource;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -30,12 +31,11 @@ import org.mockito.junit.MockitoJUnitRunner;
 @RunWith(MockitoJUnitRunner.class)
 public class DataSourceCompactionConfigAuditEntryTest
 {
-  private static final String DS_WIKI = "wiki";
   private final AuditInfo auditInfo = new AuditInfo("author", "identity", "comment", "ip");
   
   private final DataSourceCompactionConfigAuditEntry firstEntry = new DataSourceCompactionConfigAuditEntry(
-      new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.MSQ),
-      DataSourceCompactionConfig.builder().forDataSource(DS_WIKI).build(),
+      new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.MSQ, null),
+      DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
       auditInfo,
       DateTimes.nowUtc()
   );
@@ -44,8 +44,8 @@ public class DataSourceCompactionConfigAuditEntryTest
   public void testhasSameConfigWithSameBaseConfigIsTrue()
   {
     final DataSourceCompactionConfigAuditEntry secondEntry = new DataSourceCompactionConfigAuditEntry(
-        new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.MSQ),
-        DataSourceCompactionConfig.builder().forDataSource(DS_WIKI).build(),
+        new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.MSQ, null),
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
         auditInfo,
         DateTimes.nowUtc()
     );
@@ -57,8 +57,8 @@ public class DataSourceCompactionConfigAuditEntryTest
   public void testhasSameConfigWithDifferentClusterConfigIsFalse()
   {
     DataSourceCompactionConfigAuditEntry secondEntry = new DataSourceCompactionConfigAuditEntry(
-        new ClusterCompactionConfig(0.1, 9, false, CompactionEngine.MSQ),
-        DataSourceCompactionConfig.builder().forDataSource(DS_WIKI).build(),
+        new ClusterCompactionConfig(0.1, 9, false, CompactionEngine.MSQ, null),
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
         auditInfo,
         DateTimes.nowUtc()
     );
@@ -66,8 +66,8 @@ public class DataSourceCompactionConfigAuditEntryTest
     Assert.assertFalse(secondEntry.hasSameConfig(firstEntry));
 
     secondEntry = new DataSourceCompactionConfigAuditEntry(
-        new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.NATIVE),
-        DataSourceCompactionConfig.builder().forDataSource(DS_WIKI).build(),
+        new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.NATIVE, null),
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
         auditInfo,
         DateTimes.nowUtc()
     );
@@ -79,8 +79,8 @@ public class DataSourceCompactionConfigAuditEntryTest
   public void testhasSameConfigWithDifferentDatasourceConfigIsFalse()
   {
     DataSourceCompactionConfigAuditEntry secondEntry = new DataSourceCompactionConfigAuditEntry(
-        new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.NATIVE),
-        DataSourceCompactionConfig.builder().forDataSource(DS_WIKI).build(),
+        new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.NATIVE, null),
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
         auditInfo,
         DateTimes.nowUtc()
     );
@@ -92,7 +92,7 @@ public class DataSourceCompactionConfigAuditEntryTest
   public void testhasSameConfigWithNullDatasourceConfigIsFalse()
   {
     final DataSourceCompactionConfigAuditEntry secondEntry = new DataSourceCompactionConfigAuditEntry(
-        new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.NATIVE),
+        new ClusterCompactionConfig(0.1, 9, true, CompactionEngine.NATIVE, null),
         null,
         auditInfo,
         DateTimes.nowUtc()
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigHistoryTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigHistoryTest.java
index 60a638974f4..4426d58b258 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigHistoryTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigHistoryTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.server.coordinator;
 import org.apache.druid.audit.AuditInfo;
 import org.apache.druid.indexer.CompactionEngine;
 import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.segment.TestDataSource;
 import org.joda.time.DateTime;
 import org.joda.time.Period;
 import org.junit.Assert;
@@ -37,14 +38,14 @@ public class DataSourceCompactionConfigHistoryTest
 {
   private final AuditInfo auditInfo = new AuditInfo("author", "identity", "comment", "ip");
   private final DataSourceCompactionConfig wikiCompactionConfig
-      = DataSourceCompactionConfig.builder().forDataSource(DS.WIKI).build();
+      = DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build();
 
   private DataSourceCompactionConfigHistory wikiAuditHistory;
 
   @Before
   public void setup()
   {
-    wikiAuditHistory = new DataSourceCompactionConfigHistory(DS.WIKI);
+    wikiAuditHistory = new DataSourceCompactionConfigHistory(TestDataSource.WIKI);
   }
 
   @Test
@@ -94,7 +95,7 @@ public class DataSourceCompactionConfigHistoryTest
   public void testAddDeleteAnotherDatasourceConfigShouldNotAddToHistory()
   {
     final DataSourceCompactionConfig koalaCompactionConfig
-        = DataSourceCompactionConfig.builder().forDataSource(DS.KOALA).build();
+        = DataSourceCompactionConfig.builder().forDataSource(TestDataSource.KOALA).build();
 
     wikiAuditHistory.add(
         DruidCompactionConfig.empty().withDatasourceConfig(koalaCompactionConfig),
@@ -147,7 +148,7 @@ public class DataSourceCompactionConfigHistoryTest
 
     final DataSourceCompactionConfig updatedWikiConfig
         = DataSourceCompactionConfig.builder()
-                                    .forDataSource(DS.WIKI)
+                                    .forDataSource(TestDataSource.WIKI)
                                     .withSkipOffsetFromLatest(Period.hours(5))
                                     .build();
     wikiAuditHistory.add(
@@ -177,7 +178,7 @@ public class DataSourceCompactionConfigHistoryTest
     wikiAuditHistory.add(originalConfig, auditInfo, DateTimes.nowUtc());
 
     final DruidCompactionConfig updatedConfig = originalConfig.withClusterConfig(
-        new ClusterCompactionConfig(null, null, null, CompactionEngine.MSQ)
+        new ClusterCompactionConfig(null, null, null, CompactionEngine.MSQ, null)
     );
     wikiAuditHistory.add(updatedConfig, auditInfo, DateTimes.nowUtc());
 
@@ -192,10 +193,4 @@ public class DataSourceCompactionConfigHistoryTest
     Assert.assertEquals(updatedConfig.clusterConfig(), secondEntry.getGlobalConfig());
     Assert.assertFalse(firstEntry.hasSameConfig(secondEntry));
   }
-
-  private static class DS
-  {
-    static final String KOALA = "koala";
-    static final String WIKI = "wiki";
-  }
 }
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCompactionConfigTest.java
index bf13a94e008..c8ed8d9ba53 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCompactionConfigTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCompactionConfigTest.java
@@ -22,6 +22,8 @@ package org.apache.druid.server.coordinator;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.druid.indexer.CompactionEngine;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.server.compaction.NewestSegmentFirstPolicy;
 import org.joda.time.Period;
 import org.junit.Assert;
 import org.junit.Test;
@@ -50,19 +52,20 @@ public class DruidCompactionConfigTest
         Arrays.asList(
             DataSourceCompactionConfig
                 .builder()
-                .forDataSource(DS.WIKI)
+                .forDataSource(TestDataSource.WIKI)
                 .withSkipOffsetFromLatest(Period.hours(1))
                 .build(),
             DataSourceCompactionConfig
                 .builder()
-                .forDataSource(DS.KOALA)
+                .forDataSource(TestDataSource.KOALA)
                 .withSkipOffsetFromLatest(Period.hours(2))
                 .build()
         ),
         null,
         null,
         null,
-        CompactionEngine.MSQ
+        CompactionEngine.MSQ,
+        null
     );
 
     final String json = MAPPER.writeValueAsString(config);
@@ -75,7 +78,13 @@ public class DruidCompactionConfigTest
   {
     final DruidCompactionConfig config = DruidCompactionConfig.empty();
 
-    final ClusterCompactionConfig clusterConfig = new ClusterCompactionConfig(0.5, 10, false, CompactionEngine.MSQ);
+    final ClusterCompactionConfig clusterConfig = new ClusterCompactionConfig(
+        0.5,
+        10,
+        false,
+        CompactionEngine.MSQ,
+        new NewestSegmentFirstPolicy(null)
+    );
     final DruidCompactionConfig copy = config.withClusterConfig(clusterConfig);
 
     Assert.assertEquals(clusterConfig, copy.clusterConfig());
@@ -88,17 +97,15 @@ public class DruidCompactionConfigTest
     final DruidCompactionConfig config = DruidCompactionConfig.empty();
     Assert.assertTrue(config.getCompactionConfigs().isEmpty());
 
-    final DataSourceCompactionConfig dataSourceConfig
-        = DataSourceCompactionConfig.builder().forDataSource(DS.WIKI).withEngine(CompactionEngine.NATIVE).build();
-    final DruidCompactionConfig copy = config.withDatasourceConfigs(Collections.singletonList(dataSourceConfig));
+    final DataSourceCompactionConfig dataSourceConfig = DataSourceCompactionConfig
+        .builder()
+        .forDataSource(TestDataSource.WIKI)
+        .withEngine(CompactionEngine.NATIVE)
+        .build();
+    final DruidCompactionConfig copy
+        = config.withDatasourceConfigs(Collections.singletonList(dataSourceConfig));
 
     Assert.assertEquals(1, copy.getCompactionConfigs().size());
-    Assert.assertEquals(dataSourceConfig, copy.findConfigForDatasource(DS.WIKI).orNull());
-  }
-
-  private static class DS
-  {
-    static final String WIKI = "wiki";
-    static final String KOALA = "koala";
+    Assert.assertEquals(dataSourceConfig, copy.findConfigForDatasource(TestDataSource.WIKI).orNull());
   }
 }
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
index 801016eb819..1ebeb991ccb 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
@@ -50,9 +50,9 @@ import org.apache.druid.metadata.MetadataRuleManager;
 import org.apache.druid.metadata.SegmentsMetadataManager;
 import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
 import org.apache.druid.server.DruidNode;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
 import org.apache.druid.server.coordination.ServerType;
 import org.apache.druid.server.coordinator.balancer.CostBalancerStrategyFactory;
-import org.apache.druid.server.coordinator.compact.NewestSegmentFirstPolicy;
 import org.apache.druid.server.coordinator.config.CoordinatorKillConfigs;
 import org.apache.druid.server.coordinator.config.CoordinatorPeriodConfig;
 import org.apache.druid.server.coordinator.config.CoordinatorRunConfig;
@@ -96,6 +96,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
   private static final Duration LOAD_TIMEOUT = Duration.standardMinutes(15);
   private static final long COORDINATOR_START_DELAY = 1;
   private static final long COORDINATOR_PERIOD = 100;
+  private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper();
 
   private DruidCoordinator coordinator;
   private SegmentsMetadataManager segmentsMetadataManager;
@@ -113,7 +114,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
   private DruidCoordinatorConfig druidCoordinatorConfig;
   private ObjectMapper objectMapper;
   private DruidNode druidNode;
-  private NewestSegmentFirstPolicy newestSegmentFirstPolicy;
+  private CompactionStatusTracker statusTracker;
   private final LatchableServiceEmitter serviceEmitter = new LatchableServiceEmitter();
 
   @Before
@@ -147,7 +148,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
     curator.blockUntilConnected();
     curator.create().creatingParentsIfNeeded().forPath(LOADPATH);
     objectMapper = new DefaultObjectMapper();
-    newestSegmentFirstPolicy = new NewestSegmentFirstPolicy(objectMapper);
+    statusTracker = new CompactionStatusTracker(objectMapper);
     druidCoordinatorConfig = new DruidCoordinatorConfig(
         new CoordinatorRunConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD)),
         new CoordinatorPeriodConfig(null, null),
@@ -190,8 +191,8 @@ public class DruidCoordinatorTest extends CuratorTestBase
         EasyMock.createNiceMock(LookupCoordinatorManager.class),
         new TestDruidLeaderSelector(),
         null,
-        null,
-        CentralizedDatasourceSchemaConfig.create()
+        CentralizedDatasourceSchemaConfig.create(),
+        new CompactionStatusTracker(OBJECT_MAPPER)
     );
   }
 
@@ -621,8 +622,8 @@ public class DruidCoordinatorTest extends CuratorTestBase
         EasyMock.createNiceMock(LookupCoordinatorManager.class),
         new TestDruidLeaderSelector(),
         null,
-        null,
-        CentralizedDatasourceSchemaConfig.create()
+        CentralizedDatasourceSchemaConfig.create(),
+        new CompactionStatusTracker(OBJECT_MAPPER)
     );
     // Since CompactSegments is not enabled in Custom Duty Group, then CompactSegments must be created in IndexingServiceDuties
     List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
@@ -633,7 +634,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
     Assert.assertTrue(compactSegmentsDutyFromCustomGroups.isEmpty());
 
     // CompactSegments returned by this method should be created using the DruidCoordinatorConfig in the DruidCoordinator
-    CompactSegments duty = coordinator.initializeCompactSegmentsDuty(newestSegmentFirstPolicy);
+    CompactSegments duty = coordinator.initializeCompactSegmentsDuty(statusTracker);
     Assert.assertNotNull(duty);
   }
 
@@ -661,8 +662,8 @@ public class DruidCoordinatorTest extends CuratorTestBase
         EasyMock.createNiceMock(LookupCoordinatorManager.class),
         new TestDruidLeaderSelector(),
         null,
-        null,
-        CentralizedDatasourceSchemaConfig.create()
+        CentralizedDatasourceSchemaConfig.create(),
+        new CompactionStatusTracker(OBJECT_MAPPER)
     );
     // Since CompactSegments is not enabled in Custom Duty Group, then CompactSegments must be created in IndexingServiceDuties
     List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
@@ -673,7 +674,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
     Assert.assertTrue(compactSegmentsDutyFromCustomGroups.isEmpty());
 
     // CompactSegments returned by this method should be created using the DruidCoordinatorConfig in the DruidCoordinator
-    CompactSegments duty = coordinator.initializeCompactSegmentsDuty(newestSegmentFirstPolicy);
+    CompactSegments duty = coordinator.initializeCompactSegmentsDuty(statusTracker);
     Assert.assertNotNull(duty);
   }
 
@@ -683,7 +684,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
     CoordinatorCustomDutyGroup compactSegmentCustomGroup = new CoordinatorCustomDutyGroup(
         "group1",
         Duration.standardSeconds(1),
-        ImmutableList.of(new CompactSegments(null, null))
+        ImmutableList.of(new CompactSegments(statusTracker, null))
     );
     CoordinatorCustomDutyGroups customDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of(compactSegmentCustomGroup));
     coordinator = new DruidCoordinator(
@@ -701,8 +702,8 @@ public class DruidCoordinatorTest extends CuratorTestBase
         EasyMock.createNiceMock(LookupCoordinatorManager.class),
         new TestDruidLeaderSelector(),
         null,
-        null,
-        CentralizedDatasourceSchemaConfig.create()
+        CentralizedDatasourceSchemaConfig.create(),
+        new CompactionStatusTracker(OBJECT_MAPPER)
     );
     // Since CompactSegments is enabled in Custom Duty Group, then CompactSegments must not be created in IndexingServiceDuties
     List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
@@ -715,7 +716,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
     Assert.assertNotNull(compactSegmentsDutyFromCustomGroups.get(0));
 
     // CompactSegments returned by this method should be from the Custom Duty Group
-    CompactSegments duty = coordinator.initializeCompactSegmentsDuty(newestSegmentFirstPolicy);
+    CompactSegments duty = coordinator.initializeCompactSegmentsDuty(statusTracker);
     Assert.assertNotNull(duty);
   }
 
@@ -806,8 +807,8 @@ public class DruidCoordinatorTest extends CuratorTestBase
         EasyMock.createNiceMock(LookupCoordinatorManager.class),
         new TestDruidLeaderSelector(),
         null,
-        null,
-        CentralizedDatasourceSchemaConfig.create()
+        CentralizedDatasourceSchemaConfig.create(),
+        new CompactionStatusTracker(OBJECT_MAPPER)
     );
     coordinator.start();
 
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/balancer/CostBalancerStrategyTest.java b/server/src/test/java/org/apache/druid/server/coordinator/balancer/CostBalancerStrategyTest.java
index 576dfbb03d7..5113f91e65b 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/balancer/CostBalancerStrategyTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/balancer/CostBalancerStrategyTest.java
@@ -26,6 +26,7 @@ import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.GranularityType;
 import org.apache.druid.java.util.emitter.EmittingLogger;
 import org.apache.druid.java.util.metrics.StubServiceEmitter;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordination.ServerType;
 import org.apache.druid.server.coordinator.CreateDataSegments;
 import org.apache.druid.server.coordinator.ServerHolder;
@@ -51,9 +52,7 @@ import java.util.stream.IntStream;
 public class CostBalancerStrategyTest
 {
   private static final double DELTA = 1e-6;
-  private static final String DS_WIKI = "wiki";
 
-  private StubServiceEmitter serviceEmitter;
   private ExecutorService balancerExecutor;
   private CostBalancerStrategy strategy;
   private int uniqueServerId;
@@ -64,7 +63,7 @@ public class CostBalancerStrategyTest
     balancerExecutor = Execs.singleThreaded("test-balance-exec-%d");
     strategy = new CostBalancerStrategy(MoreExecutors.listeningDecorator(balancerExecutor));
 
-    serviceEmitter = new StubServiceEmitter("test-service", "host");
+    StubServiceEmitter serviceEmitter = new StubServiceEmitter("test-service", "host");
     EmittingLogger.registerEmitter(serviceEmitter);
   }
 
@@ -154,11 +153,11 @@ public class CostBalancerStrategyTest
   @Test
   public void testJointSegmentsCostSymmetry()
   {
-    final DataSegment segmentA = CreateDataSegments.ofDatasource(DS_WIKI)
+    final DataSegment segmentA = CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                                                    .forIntervals(1, Granularities.DAY)
                                                    .startingAt("2010-01-01")
                                                    .eachOfSizeInMb(100).get(0);
-    final DataSegment segmentB = CreateDataSegments.ofDatasource(DS_WIKI)
+    final DataSegment segmentB = CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                                                    .forIntervals(1, Granularities.MONTH)
                                                    .startingAt("2010-01-01")
                                                    .eachOfSizeInMb(100).get(0);
@@ -173,7 +172,7 @@ public class CostBalancerStrategyTest
   @Test
   public void testJointSegmentsCostMultipleDatasources()
   {
-    final DataSegment wikiSegment = CreateDataSegments.ofDatasource(DS_WIKI)
+    final DataSegment wikiSegment = CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                                                       .forIntervals(1, Granularities.DAY)
                                                       .startingAt("2010-01-01")
                                                       .eachOfSizeInMb(100).get(0);
@@ -245,21 +244,21 @@ public class CostBalancerStrategyTest
   {
     // Create segments for different granularities
     final List<DataSegment> daySegments =
-        CreateDataSegments.ofDatasource(DS_WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(10, Granularities.DAY)
                           .startingAt("2022-01-01")
                           .withNumPartitions(10)
                           .eachOfSizeInMb(100);
 
     final List<DataSegment> monthSegments =
-        CreateDataSegments.ofDatasource(DS_WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(10, Granularities.MONTH)
                           .startingAt("2022-03-01")
                           .withNumPartitions(10)
                           .eachOfSizeInMb(100);
 
     final List<DataSegment> yearSegments =
-        CreateDataSegments.ofDatasource(DS_WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(1, Granularities.YEAR)
                           .startingAt("2023-01-01")
                           .withNumPartitions(30)
@@ -306,7 +305,7 @@ public class CostBalancerStrategyTest
 
     // Verify costs for an ALL granularity segment
     final DataSegment allGranularitySegment =
-        CreateDataSegments.ofDatasource(DS_WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(1, Granularities.ALL)
                           .eachOfSizeInMb(100).get(0);
     verifyPlacementCost(allGranularitySegment, serverA, 1.1534173737329768e7);
@@ -326,7 +325,7 @@ public class CostBalancerStrategyTest
         new TestLoadQueuePeon()
     );
 
-    final DataSegment segment = CreateDataSegments.ofDatasource(DS_WIKI).eachOfSizeInMb(100).get(0);
+    final DataSegment segment = CreateDataSegments.ofDatasource(TestDataSource.WIKI).eachOfSizeInMb(100).get(0);
 
     // Verify that computation stats have been tracked
     strategy.findServersToLoadSegment(segment, Arrays.asList(serverA, serverB));
@@ -342,7 +341,7 @@ public class CostBalancerStrategyTest
   @Test
   public void testFindServerAfterExecutorShutdownThrowsException()
   {
-    DataSegment segment = CreateDataSegments.ofDatasource(DS_WIKI)
+    DataSegment segment = CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                                             .forIntervals(1, Granularities.DAY)
                                             .startingAt("2012-10-24")
                                             .eachOfSizeInMb(100).get(0);
@@ -386,14 +385,14 @@ public class CostBalancerStrategyTest
   )
   {
     final DataSegment segmentX =
-        CreateDataSegments.ofDatasource(DS_WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(1, granularityX.getDefaultGranularity())
                           .startingAt("2012-10-24")
                           .eachOfSizeInMb(100).get(0);
 
     long startTimeY = segmentX.getInterval().getStartMillis() + startGapMillis;
     final DataSegment segmentY =
-        CreateDataSegments.ofDatasource(DS_WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(1, granularityY.getDefaultGranularity())
                           .startingAt(startTimeY)
                           .eachOfSizeInMb(100).get(0);
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/balancer/ReservoirSegmentSamplerTest.java b/server/src/test/java/org/apache/druid/server/coordinator/balancer/ReservoirSegmentSamplerTest.java
index c438219e1b3..cc522fcb546 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/balancer/ReservoirSegmentSamplerTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/balancer/ReservoirSegmentSamplerTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.server.coordinator.balancer;
 import com.google.common.collect.Lists;
 import org.apache.druid.client.DruidServer;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordination.ServerType;
 import org.apache.druid.server.coordinator.CreateDataSegments;
 import org.apache.druid.server.coordinator.ServerHolder;
@@ -50,7 +51,7 @@ public class ReservoirSegmentSamplerTest
    * num segments = 10 x 100 days
    */
   private final List<DataSegment> segments =
-      CreateDataSegments.ofDatasource("wiki")
+      CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                         .forIntervals(100, Granularities.DAY)
                         .startingAt("2022-01-01")
                         .withNumPartitions(10)
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculatorTest.java
index 56fb316aebf..b5b06afa709 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculatorTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculatorTest.java
@@ -21,6 +21,7 @@ package org.apache.druid.server.coordinator.balancer;
 
 import org.apache.druid.client.DruidServer;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordination.ServerType;
 import org.apache.druid.server.coordinator.CreateDataSegments;
 import org.apache.druid.server.coordinator.ServerHolder;
@@ -44,7 +45,7 @@ public class SegmentToMoveCalculatorTest
    * 100 days x 100 partitions = 10,000 segments.
    */
   private static final List<DataSegment> WIKI_SEGMENTS
-      = CreateDataSegments.ofDatasource("wiki")
+      = CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(100, Granularities.DAY)
                           .withNumPartitions(100)
                           .eachOfSizeInMb(500);
@@ -53,7 +54,7 @@ public class SegmentToMoveCalculatorTest
    * 10 days * 1 partitions = 10 segments.
    */
   private static final List<DataSegment> KOALA_SEGMENTS
-      = CreateDataSegments.ofDatasource("koala")
+      = CreateDataSegments.ofDatasource(TestDataSource.KOALA)
                           .forIntervals(10, Granularities.DAY)
                           .eachOfSizeInMb(500);
 
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/compact/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/coordinator/compact/CompactionStatusTest.java
deleted file mode 100644
index 0e13f8cd0e1..00000000000
--- a/server/src/test/java/org/apache/druid/server/coordinator/compact/CompactionStatusTest.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.server.coordinator.compact;
-
-import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig;
-import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
-import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
-import org.apache.druid.indexer.partitions.HashedPartitionsSpec;
-import org.apache.druid.indexer.partitions.PartitionsSpec;
-import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
-import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Collections;
-
-public class CompactionStatusTest
-{
-  private static final String DS_WIKI = "wiki";
-
-  @Test
-  public void testFindPartitionsSpecWhenGivenIsNull()
-  {
-    final ClientCompactionTaskQueryTuningConfig tuningConfig
-        = ClientCompactionTaskQueryTuningConfig.from(null);
-    Assert.assertEquals(
-        new DynamicPartitionsSpec(null, Long.MAX_VALUE),
-        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
-    );
-  }
-
-  @Test
-  public void testFindPartitionsSpecWhenGivenIsDynamicWithNullMaxTotalRows()
-  {
-    final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, null);
-    final ClientCompactionTaskQueryTuningConfig tuningConfig
-        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
-    Assert.assertEquals(
-        new DynamicPartitionsSpec(null, Long.MAX_VALUE),
-        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
-    );
-  }
-
-  @Test
-  public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxTotalRows()
-  {
-    final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, 1000L);
-    final ClientCompactionTaskQueryTuningConfig tuningConfig
-        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
-    Assert.assertEquals(
-        partitionsSpec,
-        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
-    );
-  }
-
-  @Test
-  public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxRowsPerSegment()
-  {
-    final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(100, 1000L);
-    final ClientCompactionTaskQueryTuningConfig tuningConfig
-        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
-    Assert.assertEquals(
-        partitionsSpec,
-        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
-    );
-  }
-
-  @Test
-  public void testFindPartitionsSpecFromConfigWithDeprecatedMaxRowsPerSegmentAndMaxTotalRowsReturnGivenValues()
-  {
-    final DataSourceCompactionConfig config = new DataSourceCompactionConfig(
-        "datasource",
-        null,
-        null,
-        100,
-        null,
-        new UserCompactionTaskQueryTuningConfig(
-            null,
-            null,
-            null,
-            1000L,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null
-        ),
-        null,
-        null,
-        null,
-        null,
-        null,
-        null,
-        null
-    );
-    Assert.assertEquals(
-        new DynamicPartitionsSpec(100, 1000L),
-        CompactionStatus.findPartitionsSpecFromConfig(
-            ClientCompactionTaskQueryTuningConfig.from(config)
-        )
-    );
-  }
-
-  @Test
-  public void testFindPartitionsSpecWhenGivenIsHashed()
-  {
-    final PartitionsSpec partitionsSpec =
-        new HashedPartitionsSpec(null, 100, Collections.singletonList("dim"));
-    final ClientCompactionTaskQueryTuningConfig tuningConfig
-        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
-    Assert.assertEquals(
-        partitionsSpec,
-        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
-    );
-  }
-
-  @Test
-  public void testFindPartitionsSpecWhenGivenIsRange()
-  {
-    final PartitionsSpec partitionsSpec =
-        new DimensionRangePartitionsSpec(null, 10000, Collections.singletonList("dim"), false);
-    final ClientCompactionTaskQueryTuningConfig tuningConfig
-        = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec));
-    Assert.assertEquals(
-        partitionsSpec,
-        CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)
-    );
-  }
-
-  private static DataSourceCompactionConfig createCompactionConfig(
-      PartitionsSpec partitionsSpec
-  )
-  {
-    return new DataSourceCompactionConfig(
-        DS_WIKI,
-        null, null, null, null, createTuningConfig(partitionsSpec),
-        null, null, null, null, null, null, null
-    );
-  }
-
-  private static UserCompactionTaskQueryTuningConfig createTuningConfig(
-      PartitionsSpec partitionsSpec
-  )
-  {
-    return new UserCompactionTaskQueryTuningConfig(
-        null,
-        null, null, null, null, partitionsSpec, null, null, null,
-        null, null, null, null, null, null, null, null, null, null
-    );
-  }
-}
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
index eb03acc48bf..8f84e860dea 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
@@ -47,6 +47,7 @@ import org.apache.druid.indexer.CompactionEngine;
 import org.apache.druid.indexer.RunnerTaskState;
 import org.apache.druid.indexer.TaskLocation;
 import org.apache.druid.indexer.TaskState;
+import org.apache.druid.indexer.TaskStatus;
 import org.apache.druid.indexer.TaskStatusPlus;
 import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
 import org.apache.druid.indexer.partitions.HashedPartitionsSpec;
@@ -57,6 +58,7 @@ import org.apache.druid.java.util.common.CloseableIterators;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.JodaUtils;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.parsers.CloseableIterator;
@@ -68,6 +70,7 @@ import org.apache.druid.rpc.indexing.OverlordClient;
 import org.apache.druid.segment.incremental.OnheapIncrementalIndex;
 import org.apache.druid.segment.indexing.BatchIOConfig;
 import org.apache.druid.segment.transform.TransformSpec;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
 import org.apache.druid.server.coordinator.DruidCompactionConfig;
@@ -77,8 +80,6 @@ import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig;
 import org.apache.druid.server.coordinator.UserCompactionTaskIOConfig;
 import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig;
 import org.apache.druid.server.coordinator.UserCompactionTaskTransformConfig;
-import org.apache.druid.server.coordinator.compact.CompactionSegmentSearchPolicy;
-import org.apache.druid.server.coordinator.compact.NewestSegmentFirstPolicy;
 import org.apache.druid.server.coordinator.config.DruidCoordinatorConfig;
 import org.apache.druid.server.coordinator.stats.CoordinatorRunStats;
 import org.apache.druid.server.coordinator.stats.Stats;
@@ -86,14 +87,12 @@ import org.apache.druid.timeline.CompactionState;
 import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.SegmentTimeline;
 import org.apache.druid.timeline.TimelineObjectHolder;
-import org.apache.druid.timeline.VersionedIntervalTimeline;
 import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec;
 import org.apache.druid.timeline.partition.NumberedShardSpec;
 import org.apache.druid.timeline.partition.PartitionChunk;
 import org.apache.druid.timeline.partition.ShardSpec;
 import org.apache.druid.timeline.partition.SingleDimensionShardSpec;
 import org.apache.druid.utils.Streams;
-import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.joda.time.Period;
 import org.junit.Assert;
@@ -113,6 +112,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
@@ -130,7 +130,6 @@ public class CompactSegmentsTest
   private static final int TOTAL_SEGMENT_PER_DATASOURCE = 44;
   private static final int TOTAL_INTERVAL_PER_DATASOURCE = 11;
   private static final int MAXIMUM_CAPACITY_WITH_AUTO_SCALE = 10;
-  private static final NewestSegmentFirstPolicy SEARCH_POLICY = new NewestSegmentFirstPolicy(JSON_MAPPER);
 
   @Parameterized.Parameters(name = "scenario: {0}, engine: {2}")
   public static Collection<Object[]> constructorFeeder()
@@ -178,6 +177,7 @@ public class CompactSegmentsTest
   private final CompactionEngine engine;
 
   private DataSourcesSnapshot dataSources;
+  private CompactionStatusTracker statusTracker;
   Map<String, List<DataSegment>> datasourceToSegments = new HashMap<>();
 
   public CompactSegmentsTest(
@@ -210,6 +210,7 @@ public class CompactSegmentsTest
       }
     }
     dataSources = DataSourcesSnapshot.fromUsedSegments(allSegments, ImmutableMap.of());
+    statusTracker = new CompactionStatusTracker(JSON_MAPPER);
   }
 
   private DataSegment createSegment(String dataSource, int startDay, boolean beforeNoon, int partition)
@@ -252,10 +253,10 @@ public class CompactSegmentsTest
         new InjectableValues.Std()
             .addValue(DruidCoordinatorConfig.class, COORDINATOR_CONFIG)
             .addValue(OverlordClient.class, overlordClient)
-            .addValue(CompactionSegmentSearchPolicy.class, SEARCH_POLICY)
+            .addValue(CompactionStatusTracker.class, statusTracker)
     );
 
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
     String compactSegmentString = JSON_MAPPER.writeValueAsString(compactSegments);
     CompactSegments serdeCompactSegments = JSON_MAPPER.readValue(compactSegmentString, CompactSegments.class);
 
@@ -267,7 +268,7 @@ public class CompactSegmentsTest
   public void testRun()
   {
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
 
     final Supplier<String> expectedVersionSupplier = new Supplier<String>()
     {
@@ -343,7 +344,7 @@ public class CompactSegmentsTest
   public void testMakeStats()
   {
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
 
     // Before any compaction, we do not have any snapshot of compactions
     Map<String, AutoCompactionSnapshot> autoCompactionSnapshots = compactSegments.getAutoCompactionSnapshot();
@@ -437,7 +438,7 @@ public class CompactSegmentsTest
     dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of());
 
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
 
     // Before any compaction, we do not have any snapshot of compactions
     Map<String, AutoCompactionSnapshot> autoCompactionSnapshots = compactSegments.getAutoCompactionSnapshot();
@@ -499,7 +500,7 @@ public class CompactSegmentsTest
   public void testMakeStatsWithDeactivatedDatasource()
   {
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
 
     // Before any compaction, we do not have any snapshot of compactions
     Map<String, AutoCompactionSnapshot> autoCompactionSnapshots = compactSegments.getAutoCompactionSnapshot();
@@ -591,7 +592,7 @@ public class CompactSegmentsTest
     dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of());
 
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
 
     // Before any compaction, we do not have any snapshot of compactions
     Map<String, AutoCompactionSnapshot> autoCompactionSnapshots = compactSegments.getAutoCompactionSnapshot();
@@ -650,7 +651,7 @@ public class CompactSegmentsTest
   public void testRunMultipleCompactionTaskSlots()
   {
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
 
     final CoordinatorRunStats stats = doCompactSegments(compactSegments, 3);
     Assert.assertEquals(3, stats.get(Stats.Compaction.AVAILABLE_SLOTS));
@@ -670,7 +671,7 @@ public class CompactSegmentsTest
     int maxCompactionSlot = 3;
     Assert.assertTrue(maxCompactionSlot < MAXIMUM_CAPACITY_WITH_AUTO_SCALE);
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
     final CoordinatorRunStats stats =
         doCompactSegments(compactSegments, createCompactionConfigs(), maxCompactionSlot, true);
     Assert.assertEquals(maxCompactionSlot, stats.get(Stats.Compaction.AVAILABLE_SLOTS));
@@ -690,7 +691,7 @@ public class CompactSegmentsTest
     int maxCompactionSlot = 100;
     Assert.assertFalse(maxCompactionSlot < MAXIMUM_CAPACITY_WITH_AUTO_SCALE);
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
     final CoordinatorRunStats stats =
         doCompactSegments(compactSegments, createCompactionConfigs(), maxCompactionSlot, true);
     Assert.assertEquals(MAXIMUM_CAPACITY_WITH_AUTO_SCALE, stats.get(Stats.Compaction.AVAILABLE_SLOTS));
@@ -712,7 +713,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -770,7 +771,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -820,7 +821,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -870,7 +871,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -931,7 +932,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -984,7 +985,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -1034,7 +1035,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -1137,6 +1138,8 @@ public class CompactSegmentsTest
            .thenReturn(
                Futures.immediateFuture(
                    CloseableIterators.withEmptyBaggage(ImmutableList.of(runningConflictCompactionTask).iterator())));
+    Mockito.when(mockClient.taskStatuses(ArgumentMatchers.any()))
+           .thenReturn(Futures.immediateFuture(Collections.emptyMap()));
     Mockito.when(mockClient.findLockedIntervals(ArgumentMatchers.any()))
            .thenReturn(Futures.immediateFuture(Collections.emptyMap()));
     Mockito.when(mockClient.cancelTask(conflictTaskId))
@@ -1146,7 +1149,7 @@ public class CompactSegmentsTest
     Mockito.when(mockClient.taskPayload(ArgumentMatchers.eq(conflictTaskId)))
            .thenReturn(Futures.immediateFuture(runningConflictCompactionTaskPayload));
 
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     compactionConfigs.add(
         new DataSourceCompactionConfig(
@@ -1205,11 +1208,57 @@ public class CompactSegmentsTest
     Assert.assertEquals(expectedGranularitySpec, taskPayload.getGranularitySpec());
   }
 
+  @Test
+  public void testIntervalIsCompactedAgainWhenSegmentIsAdded()
+  {
+    final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
+
+    final String dataSource = DATA_SOURCE_PREFIX + 0;
+    final DataSourceCompactionConfig compactionConfig = DataSourceCompactionConfig
+        .builder()
+        .forDataSource(dataSource)
+        .withSkipOffsetFromLatest(Period.seconds(0))
+        .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null))
+        .build();
+
+    CoordinatorRunStats stats = doCompactSegments(
+        compactSegments,
+        ImmutableList.of(compactionConfig)
+    );
+    Assert.assertEquals(1, stats.get(Stats.Compaction.SUBMITTED_TASKS));
+    Assert.assertEquals(1, overlordClient.submittedCompactionTasks.size());
+
+    ClientCompactionTaskQuery submittedTask = overlordClient.submittedCompactionTasks.get(0);
+    Assert.assertEquals(submittedTask.getDataSource(), dataSource);
+    Assert.assertEquals(
+        Intervals.of("2017-01-09/P1D"),
+        submittedTask.getIoConfig().getInputSpec().getInterval()
+    );
+
+    // Add more data to the latest interval
+    addMoreData(dataSource, 8);
+    stats = doCompactSegments(
+        compactSegments,
+        ImmutableList.of(compactionConfig)
+    );
+    Assert.assertEquals(1, stats.get(Stats.Compaction.SUBMITTED_TASKS));
+    Assert.assertEquals(2, overlordClient.submittedCompactionTasks.size());
+
+    // Verify that the latest interval is compacted again
+    submittedTask = overlordClient.submittedCompactionTasks.get(1);
+    Assert.assertEquals(submittedTask.getDataSource(), dataSource);
+    Assert.assertEquals(
+        Intervals.of("2017-01-09/P1D"),
+        submittedTask.getIoConfig().getInputSpec().getInterval()
+    );
+  }
+
   @Test
   public void testRunParallelCompactionMultipleCompactionTaskSlots()
   {
     final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
     final CoordinatorRunStats stats;
     // Native uses maxNumConcurrentSubTasks for task slots whereas MSQ uses maxNumTasks.
     if (engine == CompactionEngine.NATIVE) {
@@ -1246,7 +1295,7 @@ public class CompactSegmentsTest
 
     // Verify that locked intervals are skipped and only one compaction task
     // is submitted for dataSource_0
-    CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, overlordClient);
+    CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
     final CoordinatorRunStats stats =
         doCompactSegments(compactSegments, createcompactionConfigsForNative(2), 4);
     Assert.assertEquals(1, stats.get(Stats.Compaction.SUBMITTED_TASKS));
@@ -1266,7 +1315,7 @@ public class CompactSegmentsTest
     NullHandling.initializeForTests();
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -1317,7 +1366,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -1370,7 +1419,7 @@ public class CompactSegmentsTest
     AggregatorFactory[] aggregatorFactories = new AggregatorFactory[] {new CountAggregatorFactory("cnt")};
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -1452,7 +1501,7 @@ public class CompactSegmentsTest
 
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     compactionConfigs.add(
         new DataSourceCompactionConfig(
@@ -1539,7 +1588,7 @@ public class CompactSegmentsTest
 
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     compactionConfigs.add(
         new DataSourceCompactionConfig(
@@ -1596,7 +1645,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -1649,7 +1698,7 @@ public class CompactSegmentsTest
   {
     final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
     final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
-    final CompactSegments compactSegments = new CompactSegments(SEARCH_POLICY, mockClient);
+    final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient);
     final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
     final String dataSource = DATA_SOURCE_PREFIX + 0;
     compactionConfigs.add(
@@ -1806,12 +1855,12 @@ public class CompactSegmentsTest
     return doCompactSegments(compactSegments, createCompactionConfigs(), numCompactionTaskSlots);
   }
 
-  private void doCompactSegments(
+  private CoordinatorRunStats doCompactSegments(
       CompactSegments compactSegments,
       List<DataSourceCompactionConfig> compactionConfigs
   )
   {
-    doCompactSegments(compactSegments, compactionConfigs, null);
+    return doCompactSegments(compactSegments, compactionConfigs, null);
   }
 
   private CoordinatorRunStats doCompactSegments(
@@ -1839,6 +1888,7 @@ public class CompactSegmentsTest
                 numCompactionTaskSlots == null ? null : 1.0, // 100% when numCompactionTaskSlots is not null
                 numCompactionTaskSlots,
                 useAutoScaleSlots,
+                null,
                 null
             )
         )
@@ -2086,6 +2136,12 @@ public class CompactSegmentsTest
       return Futures.immediateFuture(CloseableIterators.withEmptyBaggage(Collections.emptyIterator()));
     }
 
+    @Override
+    public ListenableFuture<Map<String, TaskStatus>> taskStatuses(Set<String> taskIds)
+    {
+      return Futures.immediateFuture(Collections.emptyMap());
+    }
+
     @Override
     public ListenableFuture<IndexingTotalWorkerCapacityInfo> getTotalWorkerCapacity()
     {
@@ -2093,22 +2149,15 @@ public class CompactSegmentsTest
     }
 
     private void compactSegments(
-        VersionedIntervalTimeline<String, DataSegment> timeline,
+        SegmentTimeline timeline,
         List<DataSegment> segments,
         ClientCompactionTaskQuery clientCompactionTaskQuery
     )
     {
       Preconditions.checkArgument(segments.size() > 1);
-      DateTime minStart = DateTimes.MAX, maxEnd = DateTimes.MIN;
-      for (DataSegment segment : segments) {
-        if (segment.getInterval().getStart().compareTo(minStart) < 0) {
-          minStart = segment.getInterval().getStart();
-        }
-        if (segment.getInterval().getEnd().compareTo(maxEnd) > 0) {
-          maxEnd = segment.getInterval().getEnd();
-        }
-      }
-      Interval compactInterval = new Interval(minStart, maxEnd);
+      final Interval compactInterval = JodaUtils.umbrellaInterval(
+          segments.stream().map(DataSegment::getInterval).collect(Collectors.toList())
+      );
       segments.forEach(
           segment -> timeline.remove(
               segment.getInterval(),
@@ -2262,6 +2311,8 @@ public class CompactSegmentsTest
     final ArgumentCaptor<Object> payloadCaptor = ArgumentCaptor.forClass(Object.class);
     Mockito.when(mockClient.taskStatuses(null, null, 0))
            .thenReturn(Futures.immediateFuture(CloseableIterators.withEmptyBaggage(Collections.emptyIterator())));
+    Mockito.when(mockClient.taskStatuses(ArgumentMatchers.any()))
+           .thenReturn(Futures.immediateFuture(Collections.emptyMap()));
     Mockito.when(mockClient.findLockedIntervals(ArgumentMatchers.any()))
            .thenReturn(Futures.immediateFuture(Collections.emptyMap()));
     Mockito.when(mockClient.getTotalWorkerCapacity())
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillStalePendingSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillStalePendingSegmentsTest.java
index 11ea5bd4b57..b7a481f49cf 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillStalePendingSegmentsTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillStalePendingSegmentsTest.java
@@ -30,6 +30,7 @@ import org.apache.druid.java.util.common.CloseableIterators;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.parsers.CloseableIterator;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
 import org.apache.druid.server.coordinator.DruidCoordinatorRuntimeParams;
 import org.apache.druid.server.coordinator.stats.CoordinatorRunStats;
@@ -68,10 +69,10 @@ public class KillStalePendingSegmentsTest
   @Test
   public void testRetentionStarts1DayBeforeNowWhenNoKnownTask()
   {
-    DruidCoordinatorRuntimeParams params = createParamsWithDatasources(DS.WIKI).build();
+    DruidCoordinatorRuntimeParams params = createParamsWithDatasources(TestDataSource.WIKI).build();
     killDuty.run(params);
 
-    final Interval observedKillInterval = overlordClient.observedKillIntervals.get(DS.WIKI);
+    final Interval observedKillInterval = overlordClient.observedKillIntervals.get(TestDataSource.WIKI);
     Assert.assertEquals(DateTimes.MIN, observedKillInterval.getStart());
 
     // Verify that the cutoff time is no later than 1 day ago from now
@@ -85,15 +86,15 @@ public class KillStalePendingSegmentsTest
   public void testRetentionStarts1DayBeforeEarliestActiveTask()
   {
     final DateTime startOfEarliestActiveTask = DateTimes.of("2023-01-01");
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfEarliestActiveTask, TaskState.RUNNING);
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfEarliestActiveTask.plusHours(2), TaskState.RUNNING);
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfEarliestActiveTask.plusDays(1), TaskState.RUNNING);
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfEarliestActiveTask.plusHours(3), TaskState.RUNNING);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfEarliestActiveTask, TaskState.RUNNING);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfEarliestActiveTask.plusHours(2), TaskState.RUNNING);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfEarliestActiveTask.plusDays(1), TaskState.RUNNING);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfEarliestActiveTask.plusHours(3), TaskState.RUNNING);
 
-    DruidCoordinatorRuntimeParams params = createParamsWithDatasources(DS.WIKI).build();
+    DruidCoordinatorRuntimeParams params = createParamsWithDatasources(TestDataSource.WIKI).build();
     killDuty.run(params);
 
-    final Interval observedKillInterval = overlordClient.observedKillIntervals.get(DS.WIKI);
+    final Interval observedKillInterval = overlordClient.observedKillIntervals.get(TestDataSource.WIKI);
     Assert.assertEquals(DateTimes.MIN, observedKillInterval.getStart());
     Assert.assertEquals(startOfEarliestActiveTask.minusDays(1), observedKillInterval.getEnd());
   }
@@ -102,40 +103,40 @@ public class KillStalePendingSegmentsTest
   public void testRetentionStarts1DayBeforeLatestCompletedTask()
   {
     final DateTime startOfLatestCompletedTask = DateTimes.of("2023-01-01");
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfLatestCompletedTask, TaskState.FAILED);
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfLatestCompletedTask.minusHours(2), TaskState.SUCCESS);
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfLatestCompletedTask.minusDays(2), TaskState.FAILED);
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfLatestCompletedTask.minusDays(3), TaskState.SUCCESS);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfLatestCompletedTask, TaskState.FAILED);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfLatestCompletedTask.minusHours(2), TaskState.SUCCESS);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfLatestCompletedTask.minusDays(2), TaskState.FAILED);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfLatestCompletedTask.minusDays(3), TaskState.SUCCESS);
 
-    DruidCoordinatorRuntimeParams params = createParamsWithDatasources(DS.WIKI).build();
+    DruidCoordinatorRuntimeParams params = createParamsWithDatasources(TestDataSource.WIKI).build();
     killDuty.run(params);
 
-    final Interval observedKillInterval = overlordClient.observedKillIntervals.get(DS.WIKI);
+    final Interval observedKillInterval = overlordClient.observedKillIntervals.get(TestDataSource.WIKI);
     Assert.assertEquals(DateTimes.MIN, observedKillInterval.getStart());
     Assert.assertEquals(startOfLatestCompletedTask.minusDays(1), observedKillInterval.getEnd());
 
     final CoordinatorRunStats stats = params.getCoordinatorStats();
-    Assert.assertEquals(2, stats.get(Stats.Kill.PENDING_SEGMENTS, RowKey.of(Dimension.DATASOURCE, DS.WIKI)));
+    Assert.assertEquals(2, stats.get(Stats.Kill.PENDING_SEGMENTS, RowKey.of(Dimension.DATASOURCE, TestDataSource.WIKI)));
   }
 
   @Test
   public void testRetentionStarts1DayBeforeLatestCompletedOrEarliestActiveTask()
   {
     final DateTime startOfLatestCompletedTask = DateTimes.of("2023-02-01");
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfLatestCompletedTask, TaskState.FAILED);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfLatestCompletedTask, TaskState.FAILED);
 
     final DateTime startOfEarliestActiveTask = DateTimes.of("2023-01-01");
-    overlordClient.addTaskAndSegment(DS.KOALA, startOfEarliestActiveTask, TaskState.RUNNING);
+    overlordClient.addTaskAndSegment(TestDataSource.KOALA, startOfEarliestActiveTask, TaskState.RUNNING);
 
-    DruidCoordinatorRuntimeParams params = createParamsWithDatasources(DS.WIKI, DS.KOALA).build();
+    DruidCoordinatorRuntimeParams params = createParamsWithDatasources(TestDataSource.WIKI, TestDataSource.KOALA).build();
     killDuty.run(params);
 
     DateTime earliestEligibleTask = DateTimes.earlierOf(startOfEarliestActiveTask, startOfLatestCompletedTask);
-    final Interval wikiKillInterval = overlordClient.observedKillIntervals.get(DS.WIKI);
+    final Interval wikiKillInterval = overlordClient.observedKillIntervals.get(TestDataSource.WIKI);
     Assert.assertEquals(DateTimes.MIN, wikiKillInterval.getStart());
     Assert.assertEquals(earliestEligibleTask.minusDays(1), wikiKillInterval.getEnd());
 
-    final Interval koalaKillInterval = overlordClient.observedKillIntervals.get(DS.KOALA);
+    final Interval koalaKillInterval = overlordClient.observedKillIntervals.get(TestDataSource.KOALA);
     Assert.assertEquals(DateTimes.MIN, koalaKillInterval.getStart());
     Assert.assertEquals(earliestEligibleTask.minusDays(1), wikiKillInterval.getEnd());
   }
@@ -144,32 +145,32 @@ public class KillStalePendingSegmentsTest
   public void testPendingSegmentOfDisallowedDatasourceIsNotDeleted()
   {
     DruidCoordinatorRuntimeParams params =
-        createParamsWithDatasources(DS.WIKI, DS.KOALA).withDynamicConfigs(
+        createParamsWithDatasources(TestDataSource.WIKI, TestDataSource.KOALA).withDynamicConfigs(
             CoordinatorDynamicConfig
                 .builder()
                 .withDatasourcesToNotKillPendingSegmentsIn(
-                    Collections.singleton(DS.KOALA)
+                    Collections.singleton(TestDataSource.KOALA)
                 )
                 .build()
         ).build();
 
     DateTime startOfLatestCompletedTask = DateTimes.of("2023-01-01");
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfLatestCompletedTask, TaskState.SUCCESS);
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfLatestCompletedTask.minusDays(3), TaskState.SUCCESS);
-    overlordClient.addTaskAndSegment(DS.WIKI, startOfLatestCompletedTask.minusDays(5), TaskState.SUCCESS);
-    overlordClient.addTaskAndSegment(DS.KOALA, startOfLatestCompletedTask, TaskState.SUCCESS);
-    overlordClient.addTaskAndSegment(DS.KOALA, startOfLatestCompletedTask.minusDays(3), TaskState.SUCCESS);
-    overlordClient.addTaskAndSegment(DS.KOALA, startOfLatestCompletedTask.minusDays(5), TaskState.SUCCESS);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfLatestCompletedTask, TaskState.SUCCESS);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfLatestCompletedTask.minusDays(3), TaskState.SUCCESS);
+    overlordClient.addTaskAndSegment(TestDataSource.WIKI, startOfLatestCompletedTask.minusDays(5), TaskState.SUCCESS);
+    overlordClient.addTaskAndSegment(TestDataSource.KOALA, startOfLatestCompletedTask, TaskState.SUCCESS);
+    overlordClient.addTaskAndSegment(TestDataSource.KOALA, startOfLatestCompletedTask.minusDays(3), TaskState.SUCCESS);
+    overlordClient.addTaskAndSegment(TestDataSource.KOALA, startOfLatestCompletedTask.minusDays(5), TaskState.SUCCESS);
 
     killDuty.run(params);
 
     // Verify that stale pending segments are killed in "wiki" but not in "koala"
     final CoordinatorRunStats stats = params.getCoordinatorStats();
-    Assert.assertTrue(overlordClient.observedKillIntervals.containsKey(DS.WIKI));
-    Assert.assertEquals(2, stats.get(Stats.Kill.PENDING_SEGMENTS, RowKey.of(Dimension.DATASOURCE, DS.WIKI)));
+    Assert.assertTrue(overlordClient.observedKillIntervals.containsKey(TestDataSource.WIKI));
+    Assert.assertEquals(2, stats.get(Stats.Kill.PENDING_SEGMENTS, RowKey.of(Dimension.DATASOURCE, TestDataSource.WIKI)));
 
-    Assert.assertFalse(overlordClient.observedKillIntervals.containsKey(DS.KOALA));
-    Assert.assertEquals(0, stats.get(Stats.Kill.PENDING_SEGMENTS, RowKey.of(Dimension.DATASOURCE, DS.KOALA)));
+    Assert.assertFalse(overlordClient.observedKillIntervals.containsKey(TestDataSource.KOALA));
+    Assert.assertEquals(0, stats.get(Stats.Kill.PENDING_SEGMENTS, RowKey.of(Dimension.DATASOURCE, TestDataSource.KOALA)));
   }
 
   private DruidCoordinatorRuntimeParams.Builder createParamsWithDatasources(String... datasources)
@@ -188,12 +189,6 @@ public class KillStalePendingSegmentsTest
     return builder.withUsedSegments(usedSegments);
   }
 
-  private static class DS
-  {
-    static final String WIKI = "wiki";
-    static final String KOALA = "koala";
-  }
-
   /**
    * Simulates an Overlord with a configurable list of tasks and pending segments.
    */
@@ -254,7 +249,7 @@ public class KillStalePendingSegmentsTest
         }
       }
 
-      if (remainingPendingSegments.size() > 0) {
+      if (!remainingPendingSegments.isEmpty()) {
         datasourceToPendingSegments.put(dataSource, remainingPendingSegments);
       }
 
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/loading/RoundRobinServerSelectorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/loading/RoundRobinServerSelectorTest.java
index a3c769ef96e..cc9aa92fd9a 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/loading/RoundRobinServerSelectorTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/loading/RoundRobinServerSelectorTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.server.coordinator.loading;
 import org.apache.druid.client.DruidServer;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.segment.IndexIO;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordination.ServerType;
 import org.apache.druid.server.coordinator.DruidCluster;
 import org.apache.druid.server.coordinator.ServerHolder;
@@ -38,7 +39,7 @@ public class RoundRobinServerSelectorTest
   private static final String TIER = "normal";
 
   private final DataSegment segment = new DataSegment(
-      "wiki",
+      TestDataSource.WIKI,
       Intervals.of("2022-01-01/2022-01-02"),
       "1",
       Collections.emptyMap(),
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/BroadcastDistributionRuleTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/BroadcastDistributionRuleTest.java
index 46e62fe1432..d430c864641 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/rules/BroadcastDistributionRuleTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/BroadcastDistributionRuleTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.server.coordinator.rules;
 import org.apache.druid.client.DruidServer;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordination.ServerType;
 import org.apache.druid.server.coordinator.CreateDataSegments;
 import org.apache.druid.server.coordinator.DruidCluster;
@@ -47,12 +48,11 @@ public class BroadcastDistributionRuleTest
 {
   private int serverId = 0;
 
-  private static final String DS_WIKI = "wiki";
   private static final String TIER_1 = "tier1";
   private static final String TIER_2 = "tier2";
 
   private final DataSegment wikiSegment
-      = CreateDataSegments.ofDatasource(DS_WIKI).eachOfSizeInMb(100).get(0);
+      = CreateDataSegments.ofDatasource(TestDataSource.WIKI).eachOfSizeInMb(100).get(0);
 
   @Before
   public void setUp()
@@ -73,10 +73,10 @@ public class BroadcastDistributionRuleTest
     CoordinatorRunStats stats = runRuleOnSegment(rule, wikiSegment, params);
 
     // Verify that segment is assigned to servers of all tiers
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, TestDataSource.WIKI));
     Assert.assertTrue(serverT11.isLoadingSegment(wikiSegment));
 
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_2, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_2, TestDataSource.WIKI));
     Assert.assertTrue(serverT21.isLoadingSegment(wikiSegment));
   }
 
@@ -93,7 +93,7 @@ public class BroadcastDistributionRuleTest
     CoordinatorRunStats stats = runRuleOnSegment(rule, wikiSegment, params);
 
     // Verify that serverT11 is already serving and serverT12 is loading segment
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, TestDataSource.WIKI));
     Assert.assertFalse(serverT11.isLoadingSegment(wikiSegment));
     Assert.assertTrue(serverT11.isServingSegment(wikiSegment));
     Assert.assertTrue(serverT12.isLoadingSegment(wikiSegment));
@@ -112,7 +112,7 @@ public class BroadcastDistributionRuleTest
     ForeverBroadcastDistributionRule rule = new ForeverBroadcastDistributionRule();
     CoordinatorRunStats stats = runRuleOnSegment(rule, wikiSegment, params);
 
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, TestDataSource.WIKI));
     Assert.assertTrue(activeServer.isLoadingSegment(wikiSegment));
     Assert.assertTrue(decommissioningServer.getLoadingSegments().isEmpty());
   }
@@ -133,7 +133,7 @@ public class BroadcastDistributionRuleTest
     CoordinatorRunStats stats = runRuleOnSegment(rule, wikiSegment, params);
 
     // Verify that segment is dropped only from the decommissioning server
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.DROPPED, TIER_1, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.DROPPED, TIER_1, TestDataSource.WIKI));
     Assert.assertTrue(activeServer.getPeon().getSegmentsToDrop().isEmpty());
     Assert.assertTrue(decommissioningServer.getPeon().getSegmentsToDrop().contains(wikiSegment));
   }
@@ -160,9 +160,9 @@ public class BroadcastDistributionRuleTest
     final CoordinatorRunStats stats = runRuleOnSegment(rule, wikiSegment, params);
 
     // Verify that segment is assigned to historical, broker as well as indexer
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, DS_WIKI));
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_2, DS_WIKI));
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, broker.getServer().getTier(), DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, TestDataSource.WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_2, TestDataSource.WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, broker.getServer().getTier(), TestDataSource.WIKI));
 
     Assert.assertTrue(historical.isLoadingSegment(wikiSegment));
     Assert.assertTrue(indexer.isLoadingSegment(wikiSegment));
@@ -205,13 +205,13 @@ public class BroadcastDistributionRuleTest
     final CoordinatorRunStats stats = runRuleOnSegment(rule, wikiSegment, params);
 
     // Verify that the segment is broadcast only to the eligible server
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, DS_WIKI));
-    RowKey metricKey = RowKey.with(Dimension.DATASOURCE, DS_WIKI)
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, TIER_1, TestDataSource.WIKI));
+    RowKey metricKey = RowKey.with(Dimension.DATASOURCE, TestDataSource.WIKI)
                              .with(Dimension.TIER, TIER_1)
                              .and(Dimension.DESCRIPTION, "Not enough disk space");
     Assert.assertEquals(1L, stats.get(Stats.Segments.ASSIGN_SKIPPED, metricKey));
 
-    metricKey = RowKey.with(Dimension.DATASOURCE, DS_WIKI)
+    metricKey = RowKey.with(Dimension.DATASOURCE, TestDataSource.WIKI)
                       .with(Dimension.TIER, TIER_1)
                       .and(Dimension.DESCRIPTION, "Load queue is full");
     Assert.assertEquals(1L, stats.get(Stats.Segments.ASSIGN_SKIPPED, metricKey));
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/LoadRuleTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/LoadRuleTest.java
index 1e43d89bdda..5face5b8508 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/rules/LoadRuleTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/LoadRuleTest.java
@@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordination.ServerType;
 import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
 import org.apache.druid.server.coordinator.CreateDataSegments;
@@ -65,8 +66,6 @@ import java.util.concurrent.atomic.AtomicInteger;
 @RunWith(Parameterized.class)
 public class LoadRuleTest
 {
-  private static final String DS_WIKI = "wiki";
-
   private ListeningExecutorService exec;
   private BalancerStrategy balancerStrategy;
 
@@ -112,13 +111,13 @@ public class LoadRuleTest
         .addTier(Tier.T2, server2)
         .build();
 
-    final DataSegment segment = createDataSegment(DS_WIKI);
+    final DataSegment segment = createDataSegment(TestDataSource.WIKI);
     LoadRule rule = loadForever(ImmutableMap.of(Tier.T1, 1, Tier.T2, 2));
     Assert.assertTrue(rule.shouldMatchingSegmentBeLoaded());
     CoordinatorRunStats stats = runRuleAndGetStats(rule, segment, druidCluster);
 
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, DS_WIKI));
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T2, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, TestDataSource.WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T2, TestDataSource.WIKI));
   }
 
   private CoordinatorRunStats runRuleAndGetStats(LoadRule rule, DataSegment segment, DruidCluster cluster)
@@ -168,7 +167,7 @@ public class LoadRuleTest
         .build();
 
     final LoadRule rule = loadForever(ImmutableMap.of(Tier.T1, 1));
-    final DataSegment segment = createDataSegment(DS_WIKI);
+    final DataSegment segment = createDataSegment(TestDataSource.WIKI);
     CoordinatorRunStats firstRunStats = runRuleAndGetStats(rule, segment, druidCluster);
     Assert.assertEquals(1L, firstRunStats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, segment.getDataSource()));
     Assert.assertEquals(1, server1.getLoadingSegments().size() + server2.getLoadingSegments().size());
@@ -191,7 +190,7 @@ public class LoadRuleTest
         .build();
 
     final LoadRule rule = loadForever(ImmutableMap.of(Tier.T1, 1));
-    final DataSegment segment = createDataSegment(DS_WIKI);
+    final DataSegment segment = createDataSegment(TestDataSource.WIKI);
     CoordinatorRunStats stats = runRuleAndGetStats(rule, segment, druidCluster);
 
     // Ensure that the segment is assigned to one of the historicals
@@ -199,7 +198,7 @@ public class LoadRuleTest
 
     // Ensure that the primary segment is assigned again in case the peon timed out on loading the segment
     CoordinatorRunStats statsAfterLoadPrimary = runRuleAndGetStats(rule, segment, druidCluster);
-    Assert.assertEquals(1L, statsAfterLoadPrimary.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, DS_WIKI));
+    Assert.assertEquals(1L, statsAfterLoadPrimary.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, TestDataSource.WIKI));
   }
 
   @Test
@@ -213,7 +212,7 @@ public class LoadRuleTest
         .build();
 
     final LoadRule rule = loadForever(ImmutableMap.of(Tier.T1, 1));
-    final DataSegment segment = createDataSegment(DS_WIKI);
+    final DataSegment segment = createDataSegment(TestDataSource.WIKI);
     CoordinatorRunStats stats = runRuleAndGetStats(rule, segment, druidCluster);
 
     // Ensure that the segment is assigned to one of the historicals
@@ -230,7 +229,7 @@ public class LoadRuleTest
   public void testLoadUsedSegmentsForAllSegmentGranularityAndCachingCostBalancerStrategy()
   {
     final List<DataSegment> segments =
-        CreateDataSegments.ofDatasource(DS_WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(1, Granularities.ALL)
                           .withNumPartitions(2)
                           .eachOfSizeInMb(100);
@@ -249,13 +248,13 @@ public class LoadRuleTest
         segments.get(1),
         makeCoordinatorRuntimeParams(druidCluster, segments.toArray(new DataSegment[0]))
     );
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, TestDataSource.WIKI));
   }
 
   @Test
   public void testSegmentsAreDroppedIfLoadRuleHasZeroReplicas()
   {
-    final DataSegment segment = createDataSegment(DS_WIKI);
+    final DataSegment segment = createDataSegment(TestDataSource.WIKI);
 
     final ServerHolder serverT11 = createServer(Tier.T1, segment);
     final ServerHolder serverT12 = createServer(Tier.T2, segment);
@@ -271,8 +270,8 @@ public class LoadRuleTest
     Assert.assertFalse(rule.shouldMatchingSegmentBeLoaded());
     CoordinatorRunStats stats = runRuleAndGetStats(rule, segment, druidCluster);
 
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.DROPPED, Tier.T1, DS_WIKI));
-    Assert.assertEquals(2L, stats.getSegmentStat(Stats.Segments.DROPPED, Tier.T2, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.DROPPED, Tier.T1, TestDataSource.WIKI));
+    Assert.assertEquals(2L, stats.getSegmentStat(Stats.Segments.DROPPED, Tier.T2, TestDataSource.WIKI));
   }
 
   @Test
@@ -284,18 +283,18 @@ public class LoadRuleTest
         .addTier(Tier.T1, server)
         .build();
 
-    final DataSegment segment = createDataSegment(DS_WIKI);
+    final DataSegment segment = createDataSegment(TestDataSource.WIKI);
     LoadRule rule = loadForever(ImmutableMap.of("invalidTier", 1, Tier.T1, 1));
     Assert.assertTrue(rule.shouldMatchingSegmentBeLoaded());
     CoordinatorRunStats stats = runRuleAndGetStats(rule, segment, druidCluster);
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, DS_WIKI));
-    Assert.assertEquals(0L, stats.getSegmentStat(Stats.Segments.ASSIGNED, "invalidTier", DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, TestDataSource.WIKI));
+    Assert.assertEquals(0L, stats.getSegmentStat(Stats.Segments.ASSIGNED, "invalidTier", TestDataSource.WIKI));
   }
 
   @Test
   public void testDropIgnoresInvalidTiers()
   {
-    final DataSegment segment = createDataSegment(DS_WIKI);
+    final DataSegment segment = createDataSegment(TestDataSource.WIKI);
 
     // Cluster has 1 tier with 2 servers
     ServerHolder server1 = createServer(Tier.T1, segment);
@@ -308,8 +307,8 @@ public class LoadRuleTest
     LoadRule rule = loadForever(ImmutableMap.of("invalidTier", 1, Tier.T1, 1));
     CoordinatorRunStats stats = runRuleAndGetStats(rule, segment, druidCluster);
 
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.DROPPED, Tier.T1, DS_WIKI));
-    Assert.assertEquals(0L, stats.getSegmentStat(Stats.Segments.DROPPED, "invalidTier", DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.DROPPED, Tier.T1, TestDataSource.WIKI));
+    Assert.assertEquals(0L, stats.getSegmentStat(Stats.Segments.DROPPED, "invalidTier", TestDataSource.WIKI));
   }
 
   @Test
@@ -374,11 +373,11 @@ public class LoadRuleTest
     // Load rule requires 1 replica on each tier
     LoadRule rule = loadForever(ImmutableMap.of(Tier.T1, 1, Tier.T2, 1));
     Assert.assertTrue(rule.shouldMatchingSegmentBeLoaded());
-    DataSegment segment = createDataSegment(DS_WIKI);
+    DataSegment segment = createDataSegment(TestDataSource.WIKI);
     CoordinatorRunStats stats = runRuleAndGetStats(rule, segment, druidCluster);
 
     // Verify that segment is not loaded on decommissioning server
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T2, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T2, TestDataSource.WIKI));
     Assert.assertEquals(0, decommServerT1.getLoadingSegments().size());
     Assert.assertTrue(serverT2.getLoadingSegments().contains(segment));
   }
@@ -392,7 +391,7 @@ public class LoadRuleTest
     ServerHolder serverT21 = createServer(Tier.T2);
     ServerHolder serverT22 = createServer(Tier.T2);
 
-    final DataSegment segment = createDataSegment(DS_WIKI);
+    final DataSegment segment = createDataSegment(TestDataSource.WIKI);
     DruidCluster druidCluster = DruidCluster
         .builder()
         .addTier(Tier.T1, decommServerT11, serverT12)
@@ -404,11 +403,11 @@ public class LoadRuleTest
     CoordinatorRunStats stats = runRuleAndGetStats(rule, segment, druidCluster);
 
     // Verify that no replica is assigned to decommissioning server
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T1, TestDataSource.WIKI));
     Assert.assertTrue(decommServerT11.getLoadingSegments().isEmpty());
     Assert.assertEquals(0, decommServerT11.getLoadingSegments().size());
 
-    Assert.assertEquals(2L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T2, DS_WIKI));
+    Assert.assertEquals(2L, stats.getSegmentStat(Stats.Segments.ASSIGNED, Tier.T2, TestDataSource.WIKI));
   }
 
   /**
@@ -444,7 +443,7 @@ public class LoadRuleTest
   @Test
   public void testExtraReplicasAreDroppedFromDecommissioningServer()
   {
-    final DataSegment segment1 = createDataSegment(DS_WIKI);
+    final DataSegment segment1 = createDataSegment(TestDataSource.WIKI);
 
     // 3 servers, each serving the same segment
     final ServerHolder server1 = createServer(Tier.T1, segment1);
@@ -465,7 +464,7 @@ public class LoadRuleTest
     );
 
     // Verify that the extra replica is dropped from the decommissioning server
-    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.DROPPED, Tier.T1, DS_WIKI));
+    Assert.assertEquals(1L, stats.getSegmentStat(Stats.Segments.DROPPED, Tier.T1, TestDataSource.WIKI));
     Assert.assertEquals(0, server1.getPeon().getSegmentsToDrop().size());
     Assert.assertEquals(1, server2.getPeon().getSegmentsToDrop().size());
     Assert.assertEquals(0, server3.getPeon().getSegmentsToDrop().size());
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/BalancingStrategiesTest.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/BalancingStrategiesTest.java
index 89bff979a0c..ceaeb777c8c 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/BalancingStrategiesTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/BalancingStrategiesTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.server.coordinator.simulate;
 
 import org.apache.druid.client.DruidServer;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.timeline.DataSegment;
 import org.junit.Assert;
 import org.junit.Test;
@@ -65,7 +66,7 @@ public class BalancingStrategiesTest extends CoordinatorSimulationBaseTest
     CoordinatorSimulation sim =
         CoordinatorSimulation.builder()
                              .withBalancer(strategy)
-                             .withRules(DS.WIKI, Load.on(Tier.T1, 1).forever())
+                             .withRules(TestDataSource.WIKI, Load.on(Tier.T1, 1).forever())
                              .withServers(historicals)
                              .withSegments(segments)
                              .build();
@@ -99,7 +100,7 @@ public class BalancingStrategiesTest extends CoordinatorSimulationBaseTest
     CoordinatorSimulation sim =
         CoordinatorSimulation.builder()
                              .withBalancer(strategy)
-                             .withRules(DS.WIKI, Load.on(Tier.T1, 1).forever())
+                             .withRules(TestDataSource.WIKI, Load.on(Tier.T1, 1).forever())
                              .withServers(historicals)
                              .withSegments(segments)
                              .build();
@@ -145,7 +146,7 @@ public class BalancingStrategiesTest extends CoordinatorSimulationBaseTest
     CoordinatorSimulation sim =
         CoordinatorSimulation.builder()
                              .withBalancer(strategy)
-                             .withRules(DS.WIKI, Load.on(Tier.T1, 1).forever())
+                             .withRules(TestDataSource.WIKI, Load.on(Tier.T1, 1).forever())
                              .withServers(historicals)
                              .withSegments(segments)
                              .build();
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBaseTest.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBaseTest.java
index 6fd23bd8f46..1fd3d54dc45 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBaseTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBaseTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.server.coordinator.simulate;
 import org.apache.druid.client.DruidServer;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.metrics.MetricsVerifier;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordination.ServerType;
 import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
 import org.apache.druid.server.coordinator.CreateDataSegments;
@@ -201,12 +202,6 @@ public abstract class CoordinatorSimulationBaseTest implements
 
   // Utility and constant holder classes
 
-  static class DS
-  {
-    static final String WIKI = "wiki";
-    static final String KOALA = "koala";
-  }
-
   static class Tier
   {
     static final String T1 = "tier_t1";
@@ -230,33 +225,33 @@ public abstract class CoordinatorSimulationBaseTest implements
   static class Segments
   {
     /**
-     * Segments of datasource {@link DS#WIKI}, size 500 MB each,
+     * Segments of datasource {@link TestDataSource#WIKI}, size 500 MB each,
      * spanning 1 day containing 10 partitions each.
      */
     static final List<DataSegment> WIKI_10X1D =
-        CreateDataSegments.ofDatasource(DS.WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(1, Granularities.DAY)
                           .startingAt("2022-01-01")
                           .withNumPartitions(10)
                           .eachOfSizeInMb(500);
 
     /**
-     * Segments of datasource {@link DS#WIKI}, size 500 MB each,
+     * Segments of datasource {@link TestDataSource#WIKI}, size 500 MB each,
      * spanning 100 days containing 10 partitions each.
      */
     static final List<DataSegment> WIKI_10X100D =
-        CreateDataSegments.ofDatasource(DS.WIKI)
+        CreateDataSegments.ofDatasource(TestDataSource.WIKI)
                           .forIntervals(100, Granularities.DAY)
                           .startingAt("2022-01-01")
                           .withNumPartitions(10)
                           .eachOfSizeInMb(500);
 
     /**
-     * Segments of datasource {@link DS#KOALA}, size 500 MB each,
+     * Segments of datasource {@link TestDataSource#KOALA}, size 500 MB each,
      * spanning 100 days containing 100 partitions each.
      */
     static final List<DataSegment> KOALA_100X100D =
-        CreateDataSegments.ofDatasource(DS.KOALA)
+        CreateDataSegments.ofDatasource(TestDataSource.KOALA)
                           .forIntervals(100, Granularities.DAY)
                           .startingAt("2022-01-01")
                           .withNumPartitions(100)
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java
index b32f8c055ca..f82b89c6859 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java
@@ -37,6 +37,7 @@ import org.apache.druid.java.util.http.client.HttpClient;
 import org.apache.druid.java.util.metrics.MetricsVerifier;
 import org.apache.druid.java.util.metrics.StubServiceEmitter;
 import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
 import org.apache.druid.server.coordinator.CoordinatorConfigManager;
 import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
 import org.apache.druid.server.coordinator.DruidCompactionConfig;
@@ -48,8 +49,6 @@ import org.apache.druid.server.coordinator.balancer.CachingCostBalancerStrategyF
 import org.apache.druid.server.coordinator.balancer.CostBalancerStrategyFactory;
 import org.apache.druid.server.coordinator.balancer.DiskNormalizedCostBalancerStrategyFactory;
 import org.apache.druid.server.coordinator.balancer.RandomBalancerStrategyFactory;
-import org.apache.druid.server.coordinator.compact.CompactionSegmentSearchPolicy;
-import org.apache.druid.server.coordinator.compact.NewestSegmentFirstPolicy;
 import org.apache.druid.server.coordinator.config.CoordinatorKillConfigs;
 import org.apache.druid.server.coordinator.config.CoordinatorPeriodConfig;
 import org.apache.druid.server.coordinator.config.CoordinatorRunConfig;
@@ -87,8 +86,6 @@ public class CoordinatorSimulationBuilder
               DataSegment.PruneSpecsHolder.DEFAULT
           )
       );
-  private static final CompactionSegmentSearchPolicy COMPACTION_SEGMENT_SEARCH_POLICY =
-      new NewestSegmentFirstPolicy(OBJECT_MAPPER);
   private String balancerStrategy;
   private CoordinatorDynamicConfig dynamicConfig = CoordinatorDynamicConfig.builder().build();
   private List<DruidServer> servers;
@@ -212,9 +209,9 @@ public class CoordinatorSimulationBuilder
         new CoordinatorCustomDutyGroups(Collections.emptySet()),
         env.lookupCoordinatorManager,
         env.leaderSelector,
-        COMPACTION_SEGMENT_SEARCH_POLICY,
         null,
-        CentralizedDatasourceSchemaConfig.create()
+        CentralizedDatasourceSchemaConfig.create(),
+        new CompactionStatusTracker(OBJECT_MAPPER)
     );
 
     return new SimulationImpl(coordinator, env);
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/MarkSegmentsAsUnusedTest.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/MarkSegmentsAsUnusedTest.java
index cd9ddcca1ae..f3c1a3f4fbe 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/MarkSegmentsAsUnusedTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/MarkSegmentsAsUnusedTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.server.coordinator.simulate;
 
 import org.apache.druid.client.DruidServer;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
 import org.apache.druid.timeline.DataSegment;
 import org.junit.Test;
@@ -50,7 +51,7 @@ public class MarkSegmentsAsUnusedTest extends CoordinatorSimulationBaseTest
         = CoordinatorDynamicConfig.builder().withMarkSegmentAsUnusedDelayMillis(0).build();
     final CoordinatorSimulation sim =
         CoordinatorSimulation.builder()
-                             .withRules(DS.WIKI, Load.on(Tier.T1, 0).forever())
+                             .withRules(TestDataSource.WIKI, Load.on(Tier.T1, 0).forever())
                              .withServers(servers)
                              .withSegments(segmentsV0)
                              .withDynamicConfig(dynamicConfig)
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/RoundRobinAssignmentTest.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/RoundRobinAssignmentTest.java
index 0fe884f58eb..ee78ad17ad4 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/RoundRobinAssignmentTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/RoundRobinAssignmentTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.server.coordinator.simulate;
 
 import org.apache.druid.client.DruidServer;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
 import org.apache.druid.timeline.DataSegment;
 import org.junit.Assert;
@@ -58,7 +59,7 @@ public class RoundRobinAssignmentTest extends CoordinatorSimulationBaseTest
         CoordinatorSimulation.builder()
                              .withDynamicConfig(config)
                              .withBalancer("random")
-                             .withRules(DS.WIKI, Load.on(Tier.T1, 2).forever())
+                             .withRules(TestDataSource.WIKI, Load.on(Tier.T1, 2).forever())
                              .withServers(historicals)
                              .withSegments(Segments.WIKI_10X100D)
                              .build();
@@ -92,8 +93,8 @@ public class RoundRobinAssignmentTest extends CoordinatorSimulationBaseTest
         CoordinatorSimulation.builder()
                              .withDynamicConfig(config)
                              .withBalancer("random")
-                             .withRules(DS.WIKI, Load.on(Tier.T1, 3).forever())
-                             .withRules(DS.KOALA, Load.on(Tier.T1, 1).forever())
+                             .withRules(TestDataSource.WIKI, Load.on(Tier.T1, 3).forever())
+                             .withRules(TestDataSource.KOALA, Load.on(Tier.T1, 1).forever())
                              .withServers(historicals)
                              .withSegments(segments)
                              .build();
@@ -102,8 +103,8 @@ public class RoundRobinAssignmentTest extends CoordinatorSimulationBaseTest
     // Run 1: all segments are assigned and loaded
     runCoordinatorCycle();
     loadQueuedSegments();
-    verifyValue(Metric.ASSIGNED_COUNT, filterByDatasource(DS.KOALA), 10000L);
-    verifyValue(Metric.ASSIGNED_COUNT, filterByDatasource(DS.WIKI), 3000L);
+    verifyValue(Metric.ASSIGNED_COUNT, filterByDatasource(TestDataSource.KOALA), 10000L);
+    verifyValue(Metric.ASSIGNED_COUNT, filterByDatasource(TestDataSource.WIKI), 3000L);
 
     for (DruidServer historical : historicals) {
       Assert.assertEquals(1300, historical.getTotalSegments());
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/SegmentBalancingTest.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/SegmentBalancingTest.java
index 141c6ba7a11..dabf1f2ee81 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/SegmentBalancingTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/SegmentBalancingTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.server.coordinator.simulate;
 
 import org.apache.druid.client.DruidServer;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
 import org.apache.druid.timeline.DataSegment;
 import org.junit.Assert;
@@ -37,7 +38,7 @@ public class SegmentBalancingTest extends CoordinatorSimulationBaseTest
   private DruidServer historicalT11;
   private DruidServer historicalT12;
 
-  private final String datasource = DS.WIKI;
+  private final String datasource = TestDataSource.WIKI;
   private final List<DataSegment> segments = Segments.WIKI_10X1D;
 
   @Override
@@ -245,7 +246,7 @@ public class SegmentBalancingTest extends CoordinatorSimulationBaseTest
         CoordinatorSimulation.builder()
                              .withSegments(Segments.KOALA_100X100D)
                              .withServers(historicals)
-                             .withRules(DS.KOALA, Load.on(Tier.T1, 1).forever())
+                             .withRules(TestDataSource.KOALA, Load.on(Tier.T1, 1).forever())
                              .build();
 
     startSimulation(sim);
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/SegmentLoadingTest.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/SegmentLoadingTest.java
index 4b7965e9596..7343e439c9e 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/SegmentLoadingTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/SegmentLoadingTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.server.coordinator.simulate;
 
 import org.apache.druid.client.DruidServer;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
 import org.apache.druid.timeline.DataSegment;
 import org.junit.Assert;
@@ -39,7 +40,7 @@ public class SegmentLoadingTest extends CoordinatorSimulationBaseTest
   private DruidServer historicalT21;
   private DruidServer historicalT22;
 
-  private final String datasource = DS.WIKI;
+  private final String datasource = TestDataSource.WIKI;
   private final List<DataSegment> segments = Segments.WIKI_10X1D;
 
   @Override
@@ -409,7 +410,7 @@ public class SegmentLoadingTest extends CoordinatorSimulationBaseTest
 
     // Verify that all the segments are broadcast to all historicals
     // irrespective of throttle limit
-    verifyValue(Metric.ASSIGNED_COUNT, filterByDatasource(DS.WIKI), 30L);
+    verifyValue(Metric.ASSIGNED_COUNT, filterByDatasource(datasource), 30L);
     verifyNotEmitted(Metric.DROPPED_COUNT);
   }
 
@@ -517,7 +518,7 @@ public class SegmentLoadingTest extends CoordinatorSimulationBaseTest
                              .withServers(historicalT11, historicalT12)
                              .withDynamicConfig(withReplicationThrottleLimit(100))
                              .withRules(datasource, Load.on(Tier.T1, 1).forever())
-                             .withRules(DS.KOALA, Load.on(Tier.T1, 1).forever())
+                             .withRules(TestDataSource.KOALA, Load.on(Tier.T1, 1).forever())
                              .build();
 
     startSimulation(sim);
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
index d255d0abc7d..934f61dbe07 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
@@ -47,6 +47,7 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
   private final ConcurrentMap<String, DataSegment> usedSegments = new ConcurrentHashMap<>();
 
   private volatile DataSourcesSnapshot snapshot;
+  private volatile boolean pollingStarted;
 
   public void addSegment(DataSegment segment)
   {
@@ -65,19 +66,19 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
   @Override
   public void startPollingDatabasePeriodically()
   {
-
+    this.pollingStarted = true;
   }
 
   @Override
   public void stopPollingDatabasePeriodically()
   {
-
+    this.pollingStarted = false;
   }
 
   @Override
   public boolean isPollingDatabasePeriodically()
   {
-    return true;
+    return pollingStarted;
   }
 
   @Override
@@ -227,12 +228,6 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
     return null;
   }
 
-  @Override
-  public void poll()
-  {
-
-  }
-
   @Override
   public void populateUsedFlagLastUpdatedAsync()
   {
diff --git a/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java
index 24cbfbd9462..24641c81a15 100644
--- a/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java
+++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java
@@ -37,6 +37,7 @@ import org.apache.druid.metadata.MetadataCASUpdate;
 import org.apache.druid.metadata.MetadataStorageTablesConfig;
 import org.apache.druid.metadata.TestMetadataStorageConnector;
 import org.apache.druid.metadata.TestMetadataStorageTablesConfig;
+import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.server.coordinator.ClusterCompactionConfig;
 import org.apache.druid.server.coordinator.CoordinatorConfigManager;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
@@ -109,10 +110,10 @@ public class CoordinatorCompactionConfigsResourceTest
   }
 
   @Test
-  public void testUpdateGlobalConfig()
+  public void testUpdateClusterConfig()
   {
     Response response = resource.updateClusterCompactionConfig(
-        new ClusterCompactionConfig(0.5, 10, true, CompactionEngine.MSQ),
+        new ClusterCompactionConfig(0.5, 10, true, CompactionEngine.MSQ, null),
         mockHttpServletRequest
     );
     verifyStatus(Response.Status.OK, response);
@@ -154,7 +155,7 @@ public class CoordinatorCompactionConfigsResourceTest
   @Test
   public void testGetUnknownDatasourceConfigThrowsNotFound()
   {
-    Response response = resource.getDatasourceCompactionConfig(DS.WIKI);
+    Response response = resource.getDatasourceCompactionConfig(TestDataSource.WIKI);
     verifyStatus(Response.Status.NOT_FOUND, response);
   }
 
@@ -162,12 +163,12 @@ public class CoordinatorCompactionConfigsResourceTest
   public void testAddDatasourceConfig()
   {
     final DataSourceCompactionConfig newDatasourceConfig
-        = DataSourceCompactionConfig.builder().forDataSource(DS.WIKI).build();
+        = DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build();
     Response response = resource.addOrUpdateDatasourceCompactionConfig(newDatasourceConfig, mockHttpServletRequest);
     verifyStatus(Response.Status.OK, response);
 
     final DataSourceCompactionConfig fetchedDatasourceConfig
-        = verifyAndGetPayload(resource.getDatasourceCompactionConfig(DS.WIKI), DataSourceCompactionConfig.class);
+        = verifyAndGetPayload(resource.getDatasourceCompactionConfig(TestDataSource.WIKI), DataSourceCompactionConfig.class);
     Assert.assertEquals(newDatasourceConfig, fetchedDatasourceConfig);
 
     final DruidCompactionConfig fullCompactionConfig
@@ -181,7 +182,7 @@ public class CoordinatorCompactionConfigsResourceTest
   {
     final DataSourceCompactionConfig originalDatasourceConfig = DataSourceCompactionConfig
         .builder()
-        .forDataSource(DS.WIKI)
+        .forDataSource(TestDataSource.WIKI)
         .withInputSegmentSizeBytes(500L)
         .withSkipOffsetFromLatest(Period.hours(1))
         .withGranularitySpec(
@@ -198,7 +199,7 @@ public class CoordinatorCompactionConfigsResourceTest
 
     final DataSourceCompactionConfig updatedDatasourceConfig = DataSourceCompactionConfig
         .builder()
-        .forDataSource(DS.WIKI)
+        .forDataSource(TestDataSource.WIKI)
         .withInputSegmentSizeBytes(1000L)
         .withSkipOffsetFromLatest(Period.hours(3))
         .withGranularitySpec(
@@ -211,7 +212,7 @@ public class CoordinatorCompactionConfigsResourceTest
     verifyStatus(Response.Status.OK, response);
 
     final DataSourceCompactionConfig latestDatasourceConfig
-        = verifyAndGetPayload(resource.getDatasourceCompactionConfig(DS.WIKI), DataSourceCompactionConfig.class);
+        = verifyAndGetPayload(resource.getDatasourceCompactionConfig(TestDataSource.WIKI), DataSourceCompactionConfig.class);
     Assert.assertEquals(updatedDatasourceConfig, latestDatasourceConfig);
 
     final DruidCompactionConfig fullCompactionConfig
@@ -224,21 +225,21 @@ public class CoordinatorCompactionConfigsResourceTest
   public void testDeleteDatasourceConfig()
   {
     final DataSourceCompactionConfig datasourceConfig
-        = DataSourceCompactionConfig.builder().forDataSource(DS.WIKI).build();
+        = DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build();
     Response response = resource.addOrUpdateDatasourceCompactionConfig(datasourceConfig, mockHttpServletRequest);
     verifyStatus(Response.Status.OK, response);
 
-    response = resource.deleteCompactionConfig(DS.WIKI, mockHttpServletRequest);
+    response = resource.deleteCompactionConfig(TestDataSource.WIKI, mockHttpServletRequest);
     verifyStatus(Response.Status.OK, response);
 
-    response = resource.getDatasourceCompactionConfig(DS.WIKI);
+    response = resource.getDatasourceCompactionConfig(TestDataSource.WIKI);
     verifyStatus(Response.Status.NOT_FOUND, response);
   }
 
   @Test
   public void testDeleteUnknownDatasourceConfigThrowsNotFound()
   {
-    Response response = resource.deleteCompactionConfig(DS.WIKI, mockHttpServletRequest);
+    Response response = resource.deleteCompactionConfig(TestDataSource.WIKI, mockHttpServletRequest);
     verifyStatus(Response.Status.NOT_FOUND, response);
   }
 
@@ -248,7 +249,7 @@ public class CoordinatorCompactionConfigsResourceTest
     configManager.configUpdateResult
         = ConfigManager.SetResult.retryableFailure(new Exception("retryable"));
     resource.addOrUpdateDatasourceCompactionConfig(
-        DataSourceCompactionConfig.builder().forDataSource(DS.WIKI).build(),
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
         mockHttpServletRequest
     );
 
@@ -264,7 +265,7 @@ public class CoordinatorCompactionConfigsResourceTest
     configManager.configUpdateResult
         = ConfigManager.SetResult.failure(new Exception("not retryable"));
     resource.addOrUpdateDatasourceCompactionConfig(
-        DataSourceCompactionConfig.builder().forDataSource(DS.WIKI).build(),
+        DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(),
         mockHttpServletRequest
     );
 
@@ -275,7 +276,7 @@ public class CoordinatorCompactionConfigsResourceTest
   public void testGetDatasourceConfigHistory()
   {
     final DataSourceCompactionConfig.Builder builder
-        = DataSourceCompactionConfig.builder().forDataSource(DS.WIKI);
+        = DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI);
 
     final DataSourceCompactionConfig configV1 = builder.build();
     resource.addOrUpdateDatasourceCompactionConfig(configV1, mockHttpServletRequest);
@@ -289,7 +290,7 @@ public class CoordinatorCompactionConfigsResourceTest
         .build();
     resource.addOrUpdateDatasourceCompactionConfig(configV3, mockHttpServletRequest);
 
-    Response response = resource.getCompactionConfigHistory(DS.WIKI, null, null);
+    Response response = resource.getCompactionConfigHistory(TestDataSource.WIKI, null, null);
     verifyStatus(Response.Status.OK, response);
 
     final List<DataSourceCompactionConfigAuditEntry> history
@@ -303,7 +304,7 @@ public class CoordinatorCompactionConfigsResourceTest
   @Test
   public void testGetHistoryOfUnknownDatasourceReturnsEmpty()
   {
-    Response response = resource.getCompactionConfigHistory(DS.WIKI, null, null);
+    Response response = resource.getCompactionConfigHistory(TestDataSource.WIKI, null, null);
     verifyStatus(Response.Status.OK, response);
     Assert.assertTrue(((List<?>) response.getEntity()).isEmpty());
   }
@@ -313,7 +314,7 @@ public class CoordinatorCompactionConfigsResourceTest
   {
     final DataSourceCompactionConfig datasourceConfig = DataSourceCompactionConfig
         .builder()
-        .forDataSource(DS.WIKI)
+        .forDataSource(TestDataSource.WIKI)
         .withTaskContext(Collections.singletonMap(ClientMSQContext.CTX_MAX_NUM_TASKS, 1))
         .withEngine(CompactionEngine.MSQ)
         .build();
@@ -333,14 +334,14 @@ public class CoordinatorCompactionConfigsResourceTest
   {
     final DataSourceCompactionConfig datasourceConfig = DataSourceCompactionConfig
         .builder()
-        .forDataSource(DS.WIKI)
+        .forDataSource(TestDataSource.WIKI)
         .withTaskContext(Collections.singletonMap(ClientMSQContext.CTX_MAX_NUM_TASKS, 1))
         .build();
     Response response = resource.addOrUpdateDatasourceCompactionConfig(datasourceConfig, mockHttpServletRequest);
     verifyStatus(Response.Status.OK, response);
 
     response = resource.updateClusterCompactionConfig(
-        new ClusterCompactionConfig(null, null, null, CompactionEngine.MSQ),
+        new ClusterCompactionConfig(null, null, null, CompactionEngine.MSQ, null),
         mockHttpServletRequest
     );
     verifyStatus(Response.Status.BAD_REQUEST, response);
@@ -460,21 +461,22 @@ public class CoordinatorCompactionConfigsResourceTest
           Suppliers.ofInstance(new TestConfigManagerConfig())
       );
 
-      return new TestCoordinatorConfigManager(configManager, dbConnector, tablesConfig, auditManager);
-    }
-
-    TestCoordinatorConfigManager(
-        ConfigManager configManager,
-        TestDBConnector dbConnector,
-        MetadataStorageTablesConfig tablesConfig,
-        AuditManager auditManager
-    )
-    {
-      super(
+      return new TestCoordinatorConfigManager(
           new JacksonConfigManager(configManager, OBJECT_MAPPER, auditManager),
+          configManager,
           dbConnector,
           tablesConfig
       );
+    }
+
+    TestCoordinatorConfigManager(
+        JacksonConfigManager jackson,
+        ConfigManager configManager,
+        TestDBConnector dbConnector,
+        MetadataStorageTablesConfig tablesConfig
+    )
+    {
+      super(jackson, dbConnector, tablesConfig);
       this.delegate = configManager;
     }
 
@@ -539,9 +541,4 @@ public class CoordinatorCompactionConfigsResourceTest
       return true;
     }
   }
-
-  private static class DS
-  {
-    static final String WIKI = "wiki";
-  }
 }
diff --git a/server/src/test/java/org/apache/druid/server/http/CompactionResourceTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionResourceTest.java
similarity index 53%
rename from server/src/test/java/org/apache/druid/server/http/CompactionResourceTest.java
rename to server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionResourceTest.java
index fe16d2f8c1f..8c6c747d3a8 100644
--- a/server/src/test/java/org/apache/druid/server/http/CompactionResourceTest.java
+++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionResourceTest.java
@@ -21,6 +21,11 @@ package org.apache.druid.server.http;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.druid.client.indexing.NoopOverlordClient;
+import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.server.compaction.CompactionStatistics;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.apache.druid.server.coordinator.DruidCoordinator;
 import org.easymock.EasyMock;
@@ -29,14 +34,18 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import javax.annotation.Nullable;
 import javax.ws.rs.core.Response;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
-public class CompactionResourceTest
+public class CoordinatorCompactionResourceTest
 {
   private DruidCoordinator mock;
-  private String dataSourceName = "datasource_1";
-  private AutoCompactionSnapshot expectedSnapshot = new AutoCompactionSnapshot(
+  private OverlordClient overlordClient;
+  private final String dataSourceName = "datasource_1";
+  private final AutoCompactionSnapshot expectedSnapshot = new AutoCompactionSnapshot(
       dataSourceName,
       AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING,
       1,
@@ -54,6 +63,14 @@ public class CompactionResourceTest
   public void setUp()
   {
     mock = EasyMock.createStrictMock(DruidCoordinator.class);
+    overlordClient = new NoopOverlordClient()
+    {
+      @Override
+      public ListenableFuture<Boolean> isCompactionSupervisorEnabled()
+      {
+        return Futures.immediateFuture(false);
+      }
+    };
   }
 
   @After
@@ -73,7 +90,8 @@ public class CompactionResourceTest
     EasyMock.expect(mock.getAutoCompactionSnapshot()).andReturn(expected).once();
     EasyMock.replay(mock);
 
-    final Response response = new CompactionResource(mock).getCompactionSnapshotForDataSource("");
+    final Response response = new CoordinatorCompactionResource(mock, overlordClient)
+        .getCompactionSnapshotForDataSource("");
     Assert.assertEquals(ImmutableMap.of("latestStatus", expected.values()), response.getEntity());
     Assert.assertEquals(200, response.getStatus());
   }
@@ -90,7 +108,8 @@ public class CompactionResourceTest
     EasyMock.expect(mock.getAutoCompactionSnapshot()).andReturn(expected).once();
     EasyMock.replay(mock);
 
-    final Response response = new CompactionResource(mock).getCompactionSnapshotForDataSource(null);
+    final Response response = new CoordinatorCompactionResource(mock, overlordClient)
+        .getCompactionSnapshotForDataSource(null);
     Assert.assertEquals(ImmutableMap.of("latestStatus", expected.values()), response.getEntity());
     Assert.assertEquals(200, response.getStatus());
   }
@@ -103,7 +122,8 @@ public class CompactionResourceTest
     EasyMock.expect(mock.getAutoCompactionSnapshotForDataSource(dataSourceName)).andReturn(expectedSnapshot).once();
     EasyMock.replay(mock);
 
-    final Response response = new CompactionResource(mock).getCompactionSnapshotForDataSource(dataSourceName);
+    final Response response = new CoordinatorCompactionResource(mock, overlordClient)
+        .getCompactionSnapshotForDataSource(dataSourceName);
     Assert.assertEquals(ImmutableMap.of("latestStatus", ImmutableList.of(expectedSnapshot)), response.getEntity());
     Assert.assertEquals(200, response.getStatus());
   }
@@ -116,7 +136,54 @@ public class CompactionResourceTest
     EasyMock.expect(mock.getAutoCompactionSnapshotForDataSource(dataSourceName)).andReturn(null).once();
     EasyMock.replay(mock);
 
-    final Response response = new CompactionResource(mock).getCompactionSnapshotForDataSource(dataSourceName);
+    final Response response = new CoordinatorCompactionResource(mock, overlordClient)
+        .getCompactionSnapshotForDataSource(dataSourceName);
     Assert.assertEquals(404, response.getStatus());
   }
+
+  @Test
+  public void testGetProgressForNullDatasourceReturnsBadRequest()
+  {
+    EasyMock.replay(mock);
+
+    final Response response = new CoordinatorCompactionResource(mock, overlordClient)
+        .getCompactionProgress(null);
+    Assert.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus());
+    Assert.assertEquals(
+        ImmutableMap.of("error", "No DataSource specified"),
+        response.getEntity()
+    );
+  }
+
+  @Test
+  public void testGetSnapshotWhenCompactionSupervisorIsEnabled()
+  {
+    EasyMock.replay(mock);
+
+    AutoCompactionSnapshot.Builder snapshotBuilder = AutoCompactionSnapshot.builder(dataSourceName);
+    snapshotBuilder.incrementCompactedStats(CompactionStatistics.create(100L, 10L, 1L));
+    final AutoCompactionSnapshot snapshotFromOverlord = snapshotBuilder.build();
+
+    overlordClient = new NoopOverlordClient() {
+      @Override
+      public ListenableFuture<Boolean> isCompactionSupervisorEnabled()
+      {
+        return Futures.immediateFuture(true);
+      }
+
+      @Override
+      public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+      {
+        return Futures.immediateFuture(Collections.singletonList(snapshotFromOverlord));
+      }
+    };
+
+    final Response response = new CoordinatorCompactionResource(mock, overlordClient)
+        .getCompactionSnapshotForDataSource(dataSourceName);
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(
+        Collections.singletonList(snapshotFromOverlord),
+        response.getEntity()
+    );
+  }
 }
diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
index 9b8c758a745..68923f21f1c 100644
--- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
+++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
@@ -97,12 +97,11 @@ import org.apache.druid.segment.metadata.SegmentMetadataQuerySegmentWalker;
 import org.apache.druid.segment.metadata.SegmentSchemaCache;
 import org.apache.druid.server.QueryScheduler;
 import org.apache.druid.server.QuerySchedulerProvider;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
 import org.apache.druid.server.coordinator.CoordinatorConfigManager;
 import org.apache.druid.server.coordinator.DruidCoordinator;
 import org.apache.druid.server.coordinator.MetadataManager;
 import org.apache.druid.server.coordinator.balancer.BalancerStrategyFactory;
-import org.apache.druid.server.coordinator.compact.CompactionSegmentSearchPolicy;
-import org.apache.druid.server.coordinator.compact.NewestSegmentFirstPolicy;
 import org.apache.druid.server.coordinator.config.CoordinatorKillConfigs;
 import org.apache.druid.server.coordinator.config.CoordinatorPeriodConfig;
 import org.apache.druid.server.coordinator.config.CoordinatorRunConfig;
@@ -113,8 +112,8 @@ import org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroup;
 import org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups;
 import org.apache.druid.server.coordinator.loading.LoadQueueTaskMaster;
 import org.apache.druid.server.http.ClusterResource;
-import org.apache.druid.server.http.CompactionResource;
 import org.apache.druid.server.http.CoordinatorCompactionConfigsResource;
+import org.apache.druid.server.http.CoordinatorCompactionResource;
 import org.apache.druid.server.http.CoordinatorDynamicConfigsResource;
 import org.apache.druid.server.http.CoordinatorRedirectInfo;
 import org.apache.druid.server.http.CoordinatorResource;
@@ -252,6 +251,7 @@ public class CliCoordinator extends ServerRunnable
             binder.bind(CoordinatorConfigManager.class);
             binder.bind(MetadataManager.class);
             binder.bind(DruidCoordinator.class);
+            binder.bind(CompactionStatusTracker.class).in(LazySingleton.class);
 
             LifecycleModule.register(binder, MetadataStorage.class);
             LifecycleModule.register(binder, DruidCoordinator.class);
@@ -260,7 +260,7 @@ public class CliCoordinator extends ServerRunnable
                   .to(CoordinatorJettyServerInitializer.class);
 
             Jerseys.addResource(binder, CoordinatorResource.class);
-            Jerseys.addResource(binder, CompactionResource.class);
+            Jerseys.addResource(binder, CoordinatorCompactionResource.class);
             Jerseys.addResource(binder, CoordinatorDynamicConfigsResource.class);
             Jerseys.addResource(binder, CoordinatorCompactionConfigsResource.class);
             Jerseys.addResource(binder, TiersResource.class);
@@ -286,9 +286,6 @@ public class CliCoordinator extends ServerRunnable
               );
             }
 
-            //TODO: make this configurable when there are multiple search policies
-            binder.bind(CompactionSegmentSearchPolicy.class).to(NewestSegmentFirstPolicy.class);
-
             bindAnnouncer(
                 binder,
                 Coordinator.class,
diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java
index 636d775f030..52962ddf4c1 100644
--- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java
+++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java
@@ -51,6 +51,7 @@ import org.apache.druid.guice.LifecycleModule;
 import org.apache.druid.guice.ListProvider;
 import org.apache.druid.guice.ManageLifecycle;
 import org.apache.druid.guice.PolyBind;
+import org.apache.druid.guice.SupervisorModule;
 import org.apache.druid.guice.annotations.Json;
 import org.apache.druid.indexing.common.RetryPolicyFactory;
 import org.apache.druid.indexing.common.TaskStorageDirTracker;
@@ -66,6 +67,8 @@ import org.apache.druid.indexing.common.task.batch.parallel.ParallelIndexSupervi
 import org.apache.druid.indexing.common.task.batch.parallel.ShuffleClient;
 import org.apache.druid.indexing.common.tasklogs.SwitchingTaskLogStreamer;
 import org.apache.druid.indexing.common.tasklogs.TaskRunnerTaskLogStreamer;
+import org.apache.druid.indexing.compact.CompactionScheduler;
+import org.apache.druid.indexing.compact.OverlordCompactionScheduler;
 import org.apache.druid.indexing.overlord.DruidOverlord;
 import org.apache.druid.indexing.overlord.ForkingTaskRunnerFactory;
 import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage;
@@ -91,18 +94,20 @@ import org.apache.druid.indexing.overlord.duty.TaskLogAutoCleaner;
 import org.apache.druid.indexing.overlord.duty.TaskLogAutoCleanerConfig;
 import org.apache.druid.indexing.overlord.hrtr.HttpRemoteTaskRunnerFactory;
 import org.apache.druid.indexing.overlord.hrtr.HttpRemoteTaskRunnerResource;
+import org.apache.druid.indexing.overlord.http.OverlordCompactionResource;
 import org.apache.druid.indexing.overlord.http.OverlordRedirectInfo;
 import org.apache.druid.indexing.overlord.http.OverlordResource;
 import org.apache.druid.indexing.overlord.sampler.SamplerModule;
 import org.apache.druid.indexing.overlord.setup.WorkerBehaviorConfig;
 import org.apache.druid.indexing.overlord.supervisor.SupervisorManager;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorModule;
 import org.apache.druid.indexing.overlord.supervisor.SupervisorResource;
 import org.apache.druid.indexing.worker.config.WorkerConfig;
 import org.apache.druid.indexing.worker.shuffle.DeepStorageIntermediaryDataManager;
 import org.apache.druid.indexing.worker.shuffle.IntermediaryDataManager;
 import org.apache.druid.indexing.worker.shuffle.LocalIntermediaryDataManager;
 import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.metadata.SegmentsMetadataManager;
+import org.apache.druid.metadata.SegmentsMetadataManagerProvider;
 import org.apache.druid.metadata.input.InputSourceModule;
 import org.apache.druid.query.lookup.LookupSerdeModule;
 import org.apache.druid.segment.incremental.RowIngestionMetersFactory;
@@ -111,7 +116,9 @@ import org.apache.druid.segment.realtime.ChatHandlerProvider;
 import org.apache.druid.segment.realtime.NoopChatHandlerProvider;
 import org.apache.druid.segment.realtime.appenderator.AppenderatorsManager;
 import org.apache.druid.segment.realtime.appenderator.DummyForInjectionAppenderatorsManager;
+import org.apache.druid.server.compaction.CompactionStatusTracker;
 import org.apache.druid.server.coordinator.CoordinatorOverlordServiceConfig;
+import org.apache.druid.server.coordinator.DruidCompactionConfig;
 import org.apache.druid.server.http.RedirectFilter;
 import org.apache.druid.server.http.RedirectInfo;
 import org.apache.druid.server.http.SelfDiscoveryResource;
@@ -200,7 +207,16 @@ public class CliOverlord extends ServerRunnable
               binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8090);
               binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(8290);
 
-              JsonConfigProvider.bind(binder, CentralizedDatasourceSchemaConfig.PROPERTY_PREFIX, CentralizedDatasourceSchemaConfig.class);
+              JsonConfigProvider.bind(
+                  binder,
+                  CentralizedDatasourceSchemaConfig.PROPERTY_PREFIX,
+                  CentralizedDatasourceSchemaConfig.class
+              );
+
+              binder.bind(CompactionStatusTracker.class).in(LazySingleton.class);
+              binder.bind(SegmentsMetadataManager.class)
+                    .toProvider(SegmentsMetadataManagerProvider.class)
+                    .in(ManageLifecycle.class);
             }
 
             JsonConfigProvider.bind(binder, "druid.coordinator.asOverlord", CoordinatorOverlordServiceConfig.class);
@@ -232,6 +248,7 @@ public class CliOverlord extends ServerRunnable
             binder.bind(TaskLockbox.class).in(LazySingleton.class);
             binder.bind(TaskQueryTool.class).in(LazySingleton.class);
             binder.bind(IndexerMetadataStorageAdapter.class).in(LazySingleton.class);
+            binder.bind(CompactionScheduler.class).to(OverlordCompactionScheduler.class).in(LazySingleton.class);
             binder.bind(SupervisorManager.class).in(LazySingleton.class);
 
             binder.bind(ParallelIndexSupervisorTaskClientProvider.class).toProvider(Providers.of(null));
@@ -281,6 +298,7 @@ public class CliOverlord extends ServerRunnable
             Jerseys.addResource(binder, OverlordResource.class);
             Jerseys.addResource(binder, SupervisorResource.class);
             Jerseys.addResource(binder, HttpRemoteTaskRunnerResource.class);
+            Jerseys.addResource(binder, OverlordCompactionResource.class);
 
 
             binder.bind(AppenderatorsManager.class)
@@ -371,6 +389,12 @@ public class CliOverlord extends ServerRunnable
                 binder.bind(HttpRemoteTaskRunnerFactory.class).in(LazySingleton.class);
 
                 JacksonConfigProvider.bind(binder, WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class, null);
+                JacksonConfigProvider.bind(
+                    binder,
+                    DruidCompactionConfig.CONFIG_KEY,
+                    DruidCompactionConfig.class,
+                    DruidCompactionConfig.empty()
+                );
               }
 
               @Provides
diff --git a/services/src/test/java/org/apache/druid/server/http/security/SecurityResourceFilterTest.java b/services/src/test/java/org/apache/druid/server/http/security/SecurityResourceFilterTest.java
index c0b83145d35..a23f65ff2b6 100644
--- a/services/src/test/java/org/apache/druid/server/http/security/SecurityResourceFilterTest.java
+++ b/services/src/test/java/org/apache/druid/server/http/security/SecurityResourceFilterTest.java
@@ -28,7 +28,7 @@ import org.apache.druid.server.ClientInfoResource;
 import org.apache.druid.server.QueryResource;
 import org.apache.druid.server.StatusResource;
 import org.apache.druid.server.http.BrokerResource;
-import org.apache.druid.server.http.CompactionResource;
+import org.apache.druid.server.http.CoordinatorCompactionResource;
 import org.apache.druid.server.http.CoordinatorDynamicConfigsResource;
 import org.apache.druid.server.http.CoordinatorResource;
 import org.apache.druid.server.http.DataSourcesResource;
@@ -74,7 +74,7 @@ public class SecurityResourceFilterTest extends ResourceFilterTestHelper
             getRequestPathsWithAuthorizer(SelfDiscoveryResource.class),
             getRequestPathsWithAuthorizer(BrokerQueryResource.class),
             getRequestPathsWithAuthorizer(RouterResource.class),
-            getRequestPathsWithAuthorizer(CompactionResource.class)
+            getRequestPathsWithAuthorizer(CoordinatorCompactionResource.class)
         )
     );
   }

From 088c33759d319202c9a9669453923e55fc89ba63 Mon Sep 17 00:00:00 2001
From: Jonathan Wei <jon-wei@users.noreply.github.com>
Date: Mon, 2 Sep 2024 01:35:51 -0500
Subject: [PATCH 73/99] Allow druid.azure.account to be nullable (#16960)

---
 .../org/apache/druid/storage/azure/AzureAccountConfig.java     | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java
index 938db5393eb..237b8dc7a29 100644
--- a/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java
+++ b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.Min;
-import javax.validation.constraints.NotNull;
 
 /**
  * Stores the configuration for an Azure account.
@@ -31,7 +30,7 @@ import javax.validation.constraints.NotNull;
 public class AzureAccountConfig
 {
   @JsonProperty
-  @NotNull
+  @Nullable
   private String account;
 
   /**

From 32e8e074aee7c2bf69153fd6ddad52725883b5f2 Mon Sep 17 00:00:00 2001
From: Zoltan Haindrich <kirk@rxd.hu>
Date: Mon, 2 Sep 2024 10:37:10 +0200
Subject: [PATCH 74/99] Planning could have failed if UNION ALL operator was
 completely removed (#16946)

---
 .../segment/QueryableIndexCursorHolder.java   |  2 +-
 .../builtin/CastOperatorConversion.java       |  5 +-
 .../union_removed_branch_union_nulls.iq       | 66 +++++++++++++++++++
 3 files changed, 71 insertions(+), 2 deletions(-)
 create mode 100644 sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_removed_branch_union_nulls.iq

diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
index 53412bf9da0..5188b385b36 100644
--- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
+++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorHolder.java
@@ -77,7 +77,7 @@ public class QueryableIndexCursorHolder implements CursorHolder
   @Nullable
   private final Filter filter;
   @Nullable
-  private final QueryMetrics<? extends Query> metrics;
+  private final QueryMetrics<? extends Query<?>> metrics;
   private final List<OrderBy> ordering;
   private final QueryContext queryContext;
   private final int vectorSize;
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java
index 7779d1b1747..ac34abe48d1 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java
@@ -92,10 +92,13 @@ public class CastOperatorConversion implements SqlOperatorConversion
                                               ? ExpressionType.fromColumnType(fromDruidType)
                                               : ExpressionType.fromColumnType(toDruidType);
 
-      if (fromExpressionType == null || toExpressionType == null) {
+      if (toExpressionType == null) {
         // We have no runtime type for these SQL types.
         return null;
       }
+      if (fromExpressionType == null) {
+        return DruidExpression.ofLiteral(toDruidType, DruidExpression.nullLiteral());
+      }
 
       final DruidExpression typeCastExpression;
 
diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_removed_branch_union_nulls.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_removed_branch_union_nulls.iq
new file mode 100644
index 00000000000..0711686e9ba
--- /dev/null
+++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_removed_branch_union_nulls.iq
@@ -0,0 +1,66 @@
+!set plannerStrategy DECOUPLED
+!set debug true
+!use druidtest://?numMergeBuffers=3
+!set outputformat mysql
+
+(SELECT COUNT(*), "channel" FROM "wikipedia" WHERE
+ added = '31'
+AND __time >= '2024-08-17T00:00:00+00:00' AND 'timestamp' > 1723856628000
+AND __time <= '2024-08-17T00:00:00+00:00' AND 'timestamp' <= 1723855690000
+GROUP BY "channel")
+UNION ALL
+(SELECT null,null)
+;
++--------+---------+
+| EXPR$0 | channel |
++--------+---------+
+|        |         |
++--------+---------+
+(1 row)
+
+!ok
+LogicalUnion(all=[true])
+  LogicalProject(EXPR$0=[$1], channel=[$0])
+    LogicalAggregate(group=[{0}], EXPR$0=[COUNT()])
+      LogicalProject(channel=[$1])
+        LogicalFilter(condition=[AND(=($18, CAST('31'):BIGINT NOT NULL), >=($0, CAST('2024-08-17T00:00:00+00:00'):TIMESTAMP(3) NOT NULL), >(CAST('timestamp'):BIGINT NOT NULL, 1723856628000), <=($0, CAST('2024-08-17T00:00:00+00:00'):TIMESTAMP(3) NOT NULL), <=(CAST('timestamp'):BIGINT NOT NULL, 1723855690000))])
+          LogicalTableScan(table=[[druid, wikipedia]])
+  LogicalValues(tuples=[[{ null, null }]])
+
+!convertedPlan
+DruidProject(EXPR$0=[CAST($0):BIGINT], channel=[CAST($1):VARCHAR], druid=[logical])
+  DruidValues(tuples=[[{ null, null }]])
+
+!druidPlan
+{
+  "queryType" : "scan",
+  "dataSource" : {
+    "type" : "inline",
+    "columnNames" : [ "EXPR$0", "EXPR$1" ],
+    "rows" : [ [ null, null ] ]
+  },
+  "intervals" : {
+    "type" : "intervals",
+    "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+  },
+  "virtualColumns" : [ {
+    "type" : "expression",
+    "name" : "v0",
+    "expression" : "null",
+    "outputType" : "LONG"
+  }, {
+    "type" : "expression",
+    "name" : "v1",
+    "expression" : "null",
+    "outputType" : "STRING"
+  } ],
+  "resultFormat" : "compactedList",
+  "columns" : [ "v0", "v1" ],
+  "columnTypes" : [ "LONG", "STRING" ],
+  "granularity" : {
+    "type" : "all"
+  },
+  "legacy" : false
+}
+!nativePlan
+

From a7349442e4e778792465f7af0444332ca5beecf1 Mon Sep 17 00:00:00 2001
From: Kashif Faraz <kashif.faraz@gmail.com>
Date: Mon, 2 Sep 2024 02:05:29 -0700
Subject: [PATCH 75/99] Fix computed value of maxSegmentsToMove when
 coordinator period is very low (#16984)

Bug: When coordinator period is less than 30s, `maxSegmentsToMove` is always computed as 0
irrespective of number of available threads.

Changes:
- Fix lower bound condition and set minimum value to 100.
- Add new test which fails without this fix
---
 .../server/coordinator/balancer/SegmentToMoveCalculator.java  | 4 ++--
 .../coordinator/balancer/SegmentToMoveCalculatorTest.java     | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculator.java b/server/src/main/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculator.java
index d8846740a73..01c7da44c11 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculator.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculator.java
@@ -42,7 +42,7 @@ public class SegmentToMoveCalculator
 {
   /**
    * At least this number of segments must be picked for moving in every cycle
-   * to keep the cluster well balanced.
+   * to keep the cluster well-balanced.
    */
   private static final int MIN_SEGMENTS_TO_MOVE = 100;
 
@@ -150,7 +150,7 @@ public class SegmentToMoveCalculator
     int maxComputationsInThousands = (numBalancerThreads * num30sPeriods) << 20;
     int maxSegmentsToMove = (maxComputationsInThousands / totalSegments) * 1000;
 
-    if (upperBound < lowerBound) {
+    if (upperBound < lowerBound || maxSegmentsToMove < lowerBound) {
       return Math.min(lowerBound, totalSegments);
     } else {
       return Math.min(maxSegmentsToMove, upperBound);
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculatorTest.java
index b5b06afa709..249e9638d89 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculatorTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/balancer/SegmentToMoveCalculatorTest.java
@@ -83,6 +83,10 @@ public class SegmentToMoveCalculatorTest
   @Test
   public void testMaxSegmentsToMoveIncreasesWithCoordinatorPeriod()
   {
+    Assert.assertEquals(100, computeMaxSegmentsToMoveInPeriod(200_000, Duration.millis(0)));
+    Assert.assertEquals(100, computeMaxSegmentsToMoveInPeriod(200_000, Duration.millis(10_000)));
+    Assert.assertEquals(100, computeMaxSegmentsToMoveInPeriod(200_000, Duration.millis(20_000)));
+
     Assert.assertEquals(5_000, computeMaxSegmentsToMoveInPeriod(200_000, Duration.millis(30_000)));
     Assert.assertEquals(10_000, computeMaxSegmentsToMoveInPeriod(200_000, Duration.millis(60_000)));
     Assert.assertEquals(15_000, computeMaxSegmentsToMoveInPeriod(200_000, Duration.millis(90_000)));

From ca17beb77bd02bce25334e83a8aef945064d7678 Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Tue, 3 Sep 2024 11:41:40 +0530
Subject: [PATCH 76/99] Fix issues with window functions wrt virtual columns
 when using ARRAY_CONCAT_AGG aggregator (#16971)

* Fix issues with window functions wrt virtual columns when using ARRAY_CONCAT_AGG aggregator

* Address review comment

* Address review comment
---
 .../druid/sql/calcite/rel/InputAccessor.java  |  3 +-
 .../sql/calcite/DrillWindowQueryTest.java     | 74 ++++++++++++++-----
 .../multiple_partition_columns_1.e            | 13 ++++
 .../multiple_partition_columns_1.q            |  5 ++
 .../single_partition_column_1.e               |  1 +
 .../single_partition_column_1.q               |  4 +
 .../single_partition_column_2.e               |  2 +
 .../single_partition_column_2.q               |  4 +
 .../single_partition_column_3.e               | 11 +++
 .../single_partition_column_3.q               |  4 +
 10 files changed, 100 insertions(+), 21 deletions(-)
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/multiple_partition_columns_1.e
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/multiple_partition_columns_1.q
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_1.e
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_1.q
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_2.e
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_2.q
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_3.e
 create mode 100644 sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_3.q

diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/InputAccessor.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/InputAccessor.java
index aeadab0fa9a..0a8a73c851a 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/InputAccessor.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/InputAccessor.java
@@ -29,6 +29,7 @@ import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.sql.calcite.table.RowSignatures;
 
@@ -96,7 +97,7 @@ public class InputAccessor
     if (argIndex < inputFieldCount) {
       RexInputRef inputRef = RexInputRef.of(argIndex, inputRelRowType);
       RexNode constant = predicates.constantMap.get(inputRef);
-      if (constant != null) {
+      if (constant != null && RexUtil.isLiteral(constant, false)) {
         return constant;
       }
       if (flattenedProject != null) {
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
index d9469aeec07..0cc30c49c34 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
@@ -34,6 +34,7 @@ import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.QueryRunnerFactoryConglomerate;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.join.JoinableFactoryWrapper;
 import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
 import org.apache.druid.sql.calcite.DisableUnless.DisableUnlessRule;
@@ -318,32 +319,37 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
       for (int i = 0; i < cc.size(); i++) {
         ColumnType type = rs.getColumnType(i).get();
         assertNull(type.getComplexTypeName());
-        final String val = row[i];
-        Object newVal;
-        if ("null".equals(val)) {
-          newVal = null;
-        } else {
-          switch (type.getType()) {
-            case STRING:
-              newVal = val;
-              break;
-            case LONG:
-              newVal = parseLongValue(val);
-              break;
-            case DOUBLE:
-              newVal = Numbers.parseDoubleObject(val);
-              break;
-            default:
-              throw new RuntimeException("unimplemented");
-          }
-        }
-        newRow[i] = newVal;
+        newRow[i] = parseElement(row[i], type.getType());
       }
       ret.add(newRow);
     }
     return ret;
   }
 
+  private static Object parseElement(String element, ValueType elementType)
+  {
+    if ("null".equals(element)) {
+      return null;
+    }
+    switch (elementType) {
+      case STRING:
+        return element;
+      case LONG:
+        return parseLongValue(element);
+      case DOUBLE:
+        return Numbers.parseDoubleObject(element);
+      case ARRAY:
+        String[] elements = element.substring(1, element.length() - 1).split(",");
+        List<String> arrayElements = new ArrayList<>();
+        for (String s : elements) {
+          arrayElements.add(parseElement(s.trim(), ValueType.STRING).toString());
+        }
+        return "[" + String.join(",", arrayElements) + "]";
+      default:
+        throw new RuntimeException("unimplemented type: " + elementType);
+    }
+  }
+
   private static Object parseLongValue(final String val)
   {
     if ("".equals(val)) {
@@ -7769,4 +7775,32 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
   {
     windowQueryTest();
   }
+
+  @DrillTest("druid_queries/array_concat_agg/single_partition_column_1")
+  @Test
+  public void test_array_concat_agg_with_single_partition_column_1()
+  {
+    windowQueryTest();
+  }
+
+  @DrillTest("druid_queries/array_concat_agg/single_partition_column_2")
+  @Test
+  public void test_array_concat_agg_with_single_partition_column_2()
+  {
+    windowQueryTest();
+  }
+
+  @DrillTest("druid_queries/array_concat_agg/single_partition_column_3")
+  @Test
+  public void test_array_concat_agg_with_single_partition_column_3()
+  {
+    windowQueryTest();
+  }
+
+  @DrillTest("druid_queries/array_concat_agg/multiple_partition_columns_1")
+  @Test
+  public void test_array_concat_agg_with_multiple_partition_columns_1()
+  {
+    windowQueryTest();
+  }
 }
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/multiple_partition_columns_1.e b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/multiple_partition_columns_1.e
new file mode 100644
index 00000000000..b287113e264
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/multiple_partition_columns_1.e
@@ -0,0 +1,13 @@
+Austria	null	0	[0,14,94,272,4685,6979]
+Austria	null	14	[0,14,94,272,4685,6979]
+Austria	null	94	[0,14,94,272,4685,6979]
+Austria	null	272	[0,14,94,272,4685,6979]
+Austria	null	4685	[0,14,94,272,4685,6979]
+Austria	null	6979	[0,14,94,272,4685,6979]
+Austria	Horsching	0	[0]
+Austria	Vienna	0	[0,72,93]
+Austria	Vienna	72	[0,72,93]
+Austria	Vienna	93	[0,72,93]
+Guatemala	null	0	[0]
+Guatemala	El Salvador	1	[1]
+Guatemala	Guatemala City	173	[173]
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/multiple_partition_columns_1.q b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/multiple_partition_columns_1.q
new file mode 100644
index 00000000000..d0007dfd50d
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/multiple_partition_columns_1.q
@@ -0,0 +1,5 @@
+select
+countryName, cityName, added,
+array_concat_agg(ARRAY[added], 10000) over (partition by countryName, cityName) as c1
+from wikipedia where countryName in ('Guatemala', 'Austria')
+group by countryName, cityName, added
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_1.e b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_1.e
new file mode 100644
index 00000000000..486bf8740fe
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_1.e
@@ -0,0 +1 @@
+Guatemala	["Guatemala"]
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_1.q b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_1.q
new file mode 100644
index 00000000000..5f50f240348
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_1.q
@@ -0,0 +1,4 @@
+select
+countryName, array_concat_agg(ARRAY[countryName], 10000) over (partition by countryName) as c1
+from wikipedia where countryName='Guatemala'
+group by countryName
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_2.e b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_2.e
new file mode 100644
index 00000000000..89aa6ab9bff
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_2.e
@@ -0,0 +1,2 @@
+Austria	["Austria"]
+Guatemala	["Guatemala"]
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_2.q b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_2.q
new file mode 100644
index 00000000000..c01cf7b3764
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_2.q
@@ -0,0 +1,4 @@
+select
+countryName, array_concat_agg(ARRAY[countryName], 10000) over (partition by countryName) as c1
+from wikipedia where countryName in ('Guatemala', 'Austria')
+group by countryName
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_3.e b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_3.e
new file mode 100644
index 00000000000..b41da87beaf
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_3.e
@@ -0,0 +1,11 @@
+Austria	0	[0,0]
+Guatemala	0	[0,0]
+Guatemala	1	[1]
+Austria	14	[14]
+Austria	72	[72]
+Austria	93	[93]
+Austria	94	[94]
+Guatemala	173	[173]
+Austria	272	[272]
+Austria	4685	[4685]
+Austria	6979	[6979]
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_3.q b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_3.q
new file mode 100644
index 00000000000..e781ba6199f
--- /dev/null
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/array_concat_agg/single_partition_column_3.q
@@ -0,0 +1,4 @@
+select
+countryName, added, array_concat_agg(ARRAY[added], 10000) over (partition by added) as c1
+from wikipedia where countryName in ('Guatemala', 'Austria')
+group by countryName, added

From 619d8ef964628545d03a546fb4994f36a09b2477 Mon Sep 17 00:00:00 2001
From: Sree Charan Manamala <sree.manamala@imply.io>
Date: Tue, 3 Sep 2024 11:44:52 +0530
Subject: [PATCH 77/99] Window Functions : Numeric Arrays Frame Column Writers
 - fix class cast exception (#16983)

Fix ClassCastException in ArrayFrameCoulmnWriters
---
 .../columnar/DoubleArrayFrameColumnWriter.java     |  3 +--
 .../columnar/FloatArrayFrameColumnWriter.java      |  3 +--
 .../write/columnar/LongArrayFrameColumnWriter.java |  3 +--
 .../tests/window/arrayAggWithOrderBy.sqlTest       | 14 ++++++++++++++
 4 files changed, 17 insertions(+), 6 deletions(-)
 create mode 100644 sql/src/test/resources/calcite/tests/window/arrayAggWithOrderBy.sqlTest

diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java
index 80c1b5ebfdf..607a9d6cd06 100644
--- a/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java
+++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java
@@ -51,7 +51,6 @@ public class DoubleArrayFrameColumnWriter extends NumericArrayFrameColumnWriter
   @Override
   void putArrayElement(WritableMemory memory, long offset, Number element)
   {
-    // The element is of type Double, and non-null, therefore it can be cast safely
-    memory.putDouble(offset, (double) element);
+    memory.putDouble(offset, element.doubleValue());
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java
index 47c492c28bb..d39b170cd6b 100644
--- a/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java
+++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java
@@ -51,7 +51,6 @@ public class FloatArrayFrameColumnWriter extends NumericArrayFrameColumnWriter
   @Override
   void putArrayElement(WritableMemory memory, long offset, Number element)
   {
-    // The element is of type Float, and non-null, therefore it can be cast safely
-    memory.putFloat(offset, (float) element);
+    memory.putFloat(offset, element.floatValue());
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java
index cc26fd3a36a..b545cba0b8d 100644
--- a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java
+++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java
@@ -51,7 +51,6 @@ public class LongArrayFrameColumnWriter extends NumericArrayFrameColumnWriter
   @Override
   void putArrayElement(WritableMemory memory, long offset, Number element)
   {
-    // The element is of type Long, and non-null, therefore it can be casted safely
-    memory.putLong(offset, (long) element);
+    memory.putLong(offset, element.longValue());
   }
 }
diff --git a/sql/src/test/resources/calcite/tests/window/arrayAggWithOrderBy.sqlTest b/sql/src/test/resources/calcite/tests/window/arrayAggWithOrderBy.sqlTest
new file mode 100644
index 00000000000..bee3baeac0c
--- /dev/null
+++ b/sql/src/test/resources/calcite/tests/window/arrayAggWithOrderBy.sqlTest
@@ -0,0 +1,14 @@
+type: "operatorValidation"
+
+sql: |
+  SELECT
+    count(*) over (),
+    ARRAY_AGG(d1, 100), ARRAY_AGG(f1, 100), ARRAY_AGG(m1, 100)
+  FROM numfoo
+  WHERE d1>0 AND f1>0
+  GROUP BY d1, f1, m1
+  ORDER BY d1, f1, m1
+
+expectedResults:
+  - [2,"[1.0]","[1.0]","[1.0]"]
+  - [2,"[1.7]","[0.1]","[2.0]"]

From 70bad948e379dc07911d896bfcd23b5a8a149e32 Mon Sep 17 00:00:00 2001
From: AmatyaAvadhanula <amatya.avadhanula@imply.io>
Date: Tue, 3 Sep 2024 14:58:15 +0530
Subject: [PATCH 78/99] Add integration tests for concurrent append and replace
 (#16755)

IT for streaming tasks with concurrent compaction
---
 .../clients/OverlordResourceTestClient.java   |  26 +
 .../duty/ITConcurrentAppendReplaceTest.java   | 458 ++++++++++++++++++
 .../indexer/AbstractStreamIndexingTest.java   |   3 +
 ...r_with_concurrent_locks_spec_template.json |  60 +++
 4 files changed, 547 insertions(+)
 create mode 100644 integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITConcurrentAppendReplaceTest.java
 create mode 100644 integration-tests/src/test/resources/stream/data/supervisor_with_concurrent_locks_spec_template.json

diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java
index f75dc6043f9..5a821691b2e 100644
--- a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java
+++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java
@@ -28,6 +28,7 @@ import org.apache.druid.indexer.TaskState;
 import org.apache.druid.indexer.TaskStatusPlus;
 import org.apache.druid.indexer.report.IngestionStatsAndErrorsTaskReport;
 import org.apache.druid.indexer.report.TaskReport;
+import org.apache.druid.indexing.overlord.http.TaskLockResponse;
 import org.apache.druid.indexing.overlord.http.TaskPayloadResponse;
 import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager;
 import org.apache.druid.java.util.common.ISE;
@@ -360,6 +361,31 @@ public class OverlordResourceTestClient
     }
   }
 
+  public TaskLockResponse getActiveLocks(List<LockFilterPolicy> lockFilterPolicies)
+  {
+    try {
+      String jsonBody = jsonMapper.writeValueAsString(lockFilterPolicies);
+
+      StatusResponseHolder response = httpClient.go(
+          new Request(HttpMethod.POST, new URL(getIndexerURL() + "activeLocks"))
+              .setContent(
+                  "application/json",
+                  StringUtils.toUtf8(jsonBody)
+              ),
+          StatusResponseHandler.getInstance()
+      ).get();
+      return jsonMapper.readValue(
+          response.getContent(),
+          new TypeReference<TaskLockResponse>()
+          {
+          }
+      );
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   public void waitUntilTaskCompletes(final String taskID)
   {
     waitUntilTaskCompletes(taskID, ITRetryUtil.DEFAULT_RETRY_SLEEP, ITRetryUtil.DEFAULT_RETRY_COUNT);
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITConcurrentAppendReplaceTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITConcurrentAppendReplaceTest.java
new file mode 100644
index 00000000000..94633d12366
--- /dev/null
+++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITConcurrentAppendReplaceTest.java
@@ -0,0 +1,458 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.tests.coordinator.duty;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.indexer.partitions.PartitionsSpec;
+import org.apache.druid.indexing.common.TaskLock;
+import org.apache.druid.indexing.common.TaskLockType;
+import org.apache.druid.indexing.common.task.Tasks;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.metadata.LockFilterPolicy;
+import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
+import org.apache.druid.server.coordinator.DruidCompactionConfig;
+import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig;
+import org.apache.druid.testing.clients.CompactionResourceTestClient;
+import org.apache.druid.testing.clients.TaskResponseObject;
+import org.apache.druid.testing.guice.DruidTestModuleFactory;
+import org.apache.druid.testing.utils.EventSerializer;
+import org.apache.druid.testing.utils.ITRetryUtil;
+import org.apache.druid.testing.utils.KafkaUtil;
+import org.apache.druid.testing.utils.MsqTestQueryHelper;
+import org.apache.druid.testing.utils.StreamEventWriter;
+import org.apache.druid.testing.utils.StreamGenerator;
+import org.apache.druid.testing.utils.WikipediaStreamEventStreamGenerator;
+import org.apache.druid.tests.TestNGGroup;
+import org.apache.druid.tests.indexer.AbstractKafkaIndexingServiceTest;
+import org.apache.druid.tests.indexer.AbstractStreamIndexingTest;
+import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.utils.CollectionUtils;
+import org.joda.time.Interval;
+import org.joda.time.Period;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * Integration Test to verify behaviour when there are concurrent
+ * compaction tasks with ongoing stream ingestion tasks.
+ */
+@Test(groups = {TestNGGroup.COMPACTION})
+@Guice(moduleFactory = DruidTestModuleFactory.class)
+public class ITConcurrentAppendReplaceTest extends AbstractKafkaIndexingServiceTest
+{
+  private static final Logger LOG = new Logger(ITConcurrentAppendReplaceTest.class);
+
+  @Inject
+  private CompactionResourceTestClient compactionResource;
+  
+  @Inject
+  private MsqTestQueryHelper msqHelper;
+
+  private GeneratedTestConfig generatedTestConfig;
+  private StreamGenerator streamGenerator;
+
+  private String fullDatasourceName;
+
+  private int currentRowCount = 0;
+  private boolean concurrentAppendAndReplaceLocksExisted;
+
+  @DataProvider
+  public static Object[] getKafkaTransactionStatesForTest()
+  {
+    return new Object[]{false, true};
+  }
+
+  @BeforeClass
+  public void setupClass() throws Exception
+  {
+    doBeforeClass();
+  }
+
+  @BeforeMethod
+  public void setup() throws Exception
+  {
+    generatedTestConfig = new GeneratedTestConfig(
+        Specs.PARSER_TYPE,
+        getResourceAsString(Specs.INPUT_FORMAT_PATH)
+    );
+    fullDatasourceName = generatedTestConfig.getFullDatasourceName();
+    final EventSerializer serializer = jsonMapper.readValue(
+        getResourceAsStream(Specs.SERIALIZER_PATH),
+        EventSerializer.class
+    );
+    streamGenerator = new WikipediaStreamEventStreamGenerator(serializer, 6, 100);
+    concurrentAppendAndReplaceLocksExisted = false;
+  }
+
+  @Override
+  public String getTestNamePrefix()
+  {
+    return "autocompact_lock_contention";
+  }
+
+  @Test(dataProvider = "getKafkaTransactionStatesForTest")
+  public void testConcurrentStreamAppendReplace(boolean transactionEnabled) throws Exception
+  {
+    if (shouldSkipTest(transactionEnabled)) {
+      return;
+    }
+
+    try (
+        final Closeable closer = createResourceCloser(generatedTestConfig);
+        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)
+    ) {
+      // Start supervisor
+      final String taskSpec
+          = generatedTestConfig.getStreamIngestionPropsTransform()
+                               .apply(getResourceAsString(SUPERVISOR_WITH_CONCURRENT_LOCKS_SPEC_TEMPLATE_PATH));
+      generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
+      LOG.info("supervisorSpec: [%s]", taskSpec);
+
+      // Generate data for minutes 1, 2 and 3
+      final Interval minute1 = Intervals.of("2000-01-01T01:01:00Z/2000-01-01T01:02:00Z");
+      long rowsForMinute1 = generateData(minute1, streamEventWriter);
+
+      final Interval minute2 = Intervals.of("2000-01-01T01:02:00Z/2000-01-01T01:03:00Z");
+      long rowsForMinute2 = generateData(minute2, streamEventWriter);
+
+      final Interval minute3 = Intervals.of("2000-01-01T01:03:00Z/2000-01-01T01:04:00Z");
+      long rowsForMinute3 = generateData(minute3, streamEventWriter);
+
+      Function<String, AggregatorFactory> function = name -> new LongSumAggregatorFactory(name, "count");
+
+      // Wait for data to be ingested for all the minutes
+      ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
+
+      ensureSegmentsLoaded(fullDatasourceName);
+
+      // 2 segments for each minute, total 6
+      ensureSegmentsCount(fullDatasourceName, 6);
+
+      // Trigger auto compaction
+      submitAndVerifyCompactionConfig(fullDatasourceName, null);
+      compactionResource.forceTriggerAutoCompaction();
+
+      // Verify that all the segments are now compacted
+      ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
+      ensureSegmentsLoaded(fullDatasourceName);
+      ensureSegmentsCount(fullDatasourceName, 3);
+      verifyCompactedIntervals(fullDatasourceName, minute1, minute2, minute3);
+
+      // Concurrent compaction with configured segment granularity
+      for (int i = 0; i < 5; i++) {
+        rowsForMinute1 += generateData(minute1, streamEventWriter);
+        rowsForMinute2 += generateData(minute2, streamEventWriter);
+        rowsForMinute3 += generateData(minute3, streamEventWriter);
+
+        compactionResource.forceTriggerAutoCompaction();
+        ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
+        checkAndSetConcurrentLocks(fullDatasourceName);
+      }
+
+      // Verify the state with minute granularity
+      ensureSegmentsCount(fullDatasourceName, 3);
+      ensureSegmentsLoaded(fullDatasourceName);
+      ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
+      verifyCompactedIntervals(fullDatasourceName, minute1, minute2, minute3);
+
+
+      // Use ALL segment granularity for compaction and run concurrent streaming ingestion
+      submitAndVerifyCompactionConfig(fullDatasourceName, Granularities.DAY);
+
+      for (int i = 0; i < 5; i++) {
+        rowsForMinute1 += generateData(minute1, streamEventWriter);
+        rowsForMinute2 += generateData(minute2, streamEventWriter);
+        rowsForMinute3 += generateData(minute3, streamEventWriter);
+
+        compactionResource.forceTriggerAutoCompaction();
+        ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
+        checkAndSetConcurrentLocks(fullDatasourceName);
+      }
+
+      // Verify the state with all granularity
+      ensureSegmentsCount(fullDatasourceName, 1);
+      ensureSegmentsLoaded(fullDatasourceName);
+      ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
+      verifyCompactedIntervals(fullDatasourceName, Intervals.of("2000-01-01/2000-01-02"));
+
+      Assert.assertTrue(concurrentAppendAndReplaceLocksExisted);
+    }
+  }
+
+    /**
+     * Retries until the segment count is as expected.
+     */
+  private void ensureSegmentsCount(String datasource, int numExpectedSegments)
+  {
+    ITRetryUtil.retryUntilTrue(
+        () -> {
+          compactionResource.forceTriggerAutoCompaction();
+          printTaskStatuses(datasource);
+          checkAndSetConcurrentLocks(datasource);
+          List<DataSegment> segments = coordinator.getFullSegmentsMetadata(datasource);
+          StringBuilder sb = new StringBuilder();
+          segments.forEach(
+              seg -> sb.append("{")
+                       .append(seg.getId())
+                       .append(", ")
+                       .append(seg.getSize())
+                       .append("}, ")
+          );
+          LOG.info("Found Segments: %s", sb);
+          LOG.info("Current metadata segment count: %d, expected: %d", segments.size(), numExpectedSegments);
+          return segments.size() == numExpectedSegments;
+        },
+        "Segment count check"
+    );
+  }
+
+  /**
+   * Verifies that the given intervals have been compacted.
+   */
+  private void verifyCompactedIntervals(String datasource, Interval... compactedIntervals)
+  {
+    List<DataSegment> segments = coordinator.getFullSegmentsMetadata(datasource);
+    List<DataSegment> observedCompactedSegments = new ArrayList<>();
+    Set<Interval> observedCompactedIntervals = new HashSet<>();
+    for (DataSegment segment : segments) {
+      if (segment.getLastCompactionState() != null) {
+        observedCompactedSegments.add(segment);
+        observedCompactedIntervals.add(segment.getInterval());
+      }
+    }
+
+    Set<Interval> expectedCompactedIntervals = new HashSet<>(Arrays.asList(compactedIntervals));
+    Assert.assertEquals(observedCompactedIntervals, expectedCompactedIntervals);
+
+    DynamicPartitionsSpec expectedPartitionSpec = new DynamicPartitionsSpec(
+        PartitionsSpec.DEFAULT_MAX_ROWS_PER_SEGMENT,
+        Long.MAX_VALUE
+    );
+    for (DataSegment compactedSegment : observedCompactedSegments) {
+      Assert.assertNotNull(compactedSegment.getLastCompactionState());
+      Assert.assertEquals(
+          compactedSegment.getLastCompactionState().getPartitionsSpec(),
+          expectedPartitionSpec
+      );
+    }
+  }
+
+  /**
+   * Generates data points for the specified interval.
+   *
+   * @return Number of rows generated.
+   */
+  private long generateData(Interval interval, StreamEventWriter streamEventWriter)
+  {
+    long rowCount = streamGenerator.run(
+        generatedTestConfig.getStreamName(),
+        streamEventWriter,
+        20,
+        interval.getStart()
+    );
+    LOG.info("Generated %d Rows for Interval [%s]", rowCount, interval);
+
+    return rowCount;
+  }
+
+  /**
+   * Retries until segments have been loaded.
+   */
+  private void ensureSegmentsLoaded(String datasource)
+  {
+    ITRetryUtil.retryUntilTrue(
+        () -> coordinator.areSegmentsLoaded(datasource),
+        "Segment Loading"
+    );
+  }
+
+  private void checkAndSetConcurrentLocks(String datasource)
+  {
+    LockFilterPolicy lockFilterPolicy = new LockFilterPolicy(datasource, 0, null, null);
+    final List<TaskLock> locks = indexer.getActiveLocks(ImmutableList.of(lockFilterPolicy))
+                                        .getDatasourceToLocks()
+                                        .get(datasource);
+    if (CollectionUtils.isNullOrEmpty(locks)) {
+      return;
+    }
+    Set<Interval> replaceIntervals = new HashSet<>();
+    Set<Interval> appendIntervals = new HashSet<>();
+    for (TaskLock lock : locks) {
+      if (lock.getType() == TaskLockType.APPEND) {
+        appendIntervals.add(lock.getInterval());
+      } else if (lock.getType() == TaskLockType.REPLACE) {
+        replaceIntervals.add(lock.getInterval());
+      }
+    }
+
+    for (Interval appendInterval : appendIntervals) {
+      for (Interval replaceInterval : replaceIntervals) {
+        if (appendInterval.overlaps(replaceInterval)) {
+          concurrentAppendAndReplaceLocksExisted = true;
+          return;
+        }
+      }
+    }
+  }
+
+  /**
+   * Checks if a test should be skipped based on whether transaction is enabled or not.
+   */
+  private boolean shouldSkipTest(boolean testEnableTransaction)
+  {
+    Map<String, String> kafkaTestProps = KafkaUtil
+        .getAdditionalKafkaTestConfigFromProperties(config);
+    boolean configEnableTransaction = Boolean.parseBoolean(
+        kafkaTestProps.getOrDefault(KafkaUtil.TEST_CONFIG_TRANSACTION_ENABLED, "false")
+    );
+
+    return configEnableTransaction != testEnableTransaction;
+  }
+
+  /**
+   * Submits a compaction config for the current datasource.
+   */
+  private void submitAndVerifyCompactionConfig(String datasource, Granularity segmentGranularity) throws Exception
+  {
+    final UserCompactionTaskGranularityConfig granularitySpec =
+        new UserCompactionTaskGranularityConfig(segmentGranularity, null, null);
+    final DataSourceCompactionConfig compactionConfig =
+        DataSourceCompactionConfig.builder()
+                                  .forDataSource(datasource)
+                                  .withSkipOffsetFromLatest(Period.ZERO)
+                                  .withGranularitySpec(granularitySpec)
+                                  .withTaskContext(ImmutableMap.of(Tasks.USE_CONCURRENT_LOCKS, true))
+                                  .build();
+    compactionResource.updateCompactionTaskSlot(0.5, 10, null);
+    compactionResource.submitCompactionConfig(compactionConfig);
+
+    // Wait for compaction config to persist
+    Thread.sleep(2000);
+
+    // Verify that the compaction config is updated correctly.
+    DruidCompactionConfig druidCompactionConfig = compactionResource.getCompactionConfig();
+    DataSourceCompactionConfig observedCompactionConfig = null;
+    for (DataSourceCompactionConfig dataSourceCompactionConfig : druidCompactionConfig.getCompactionConfigs()) {
+      if (dataSourceCompactionConfig.getDataSource().equals(datasource)) {
+        observedCompactionConfig = dataSourceCompactionConfig;
+      }
+    }
+    Assert.assertEquals(observedCompactionConfig, compactionConfig);
+
+    observedCompactionConfig = compactionResource.getDataSourceCompactionConfig(datasource);
+    Assert.assertEquals(observedCompactionConfig, compactionConfig);
+  }
+
+  /**
+   * Gets the number of complete compaction tasks.
+   */
+  private void printTaskStatuses(String datsource)
+  {
+    List<TaskResponseObject> incompleteTasks = indexer
+        .getUncompletedTasksForDataSource(datsource);
+    List<TaskResponseObject> completeTasks = indexer
+        .getCompleteTasksForDataSource(datsource);
+
+    printTasks(incompleteTasks, "Incomplete");
+    printTasks(completeTasks, "Complete");
+  }
+
+  private void printTasks(List<TaskResponseObject> tasks, String taskState)
+  {
+    StringBuilder sb = new StringBuilder();
+    tasks.forEach(
+        task -> sb.append("{")
+                  .append(task.getType())
+                  .append(", ")
+                  .append(task.getStatus())
+                  .append(", ")
+                  .append(task.getCreatedTime())
+                  .append("}, ")
+    );
+    LOG.info("%s Tasks: %s", taskState, sb);
+  }
+
+  /**
+   * Retries until the total row count is as expected.
+   * Also verifies that row count is non-decreasing
+   */
+  private void ensureRowCount(String datasource, long totalRows, Function<String, AggregatorFactory> function)
+  {
+    LOG.info("Verifying Row Count. Expected: %s", totalRows);
+    ITRetryUtil.retryUntilTrue(
+        () -> {
+          compactionResource.forceTriggerAutoCompaction();
+          printTaskStatuses(datasource);
+          checkAndSetConcurrentLocks(datasource);
+          int newRowCount = this.queryHelper.countRows(
+              datasource,
+              Intervals.ETERNITY,
+              function
+          );
+          Assert.assertTrue(newRowCount >= currentRowCount, "Number of events queried must be non decreasing");
+          currentRowCount = newRowCount;
+          return currentRowCount == totalRows;
+        },
+        StringUtils.format(
+            "dataSource[%s] has [%,d] queryable rows, expected [%,d]",
+            datasource,
+            this.queryHelper.countRows(
+                datasource,
+                Intervals.ETERNITY,
+                function
+            ),
+            totalRows
+        )
+    );
+  }
+
+  /**
+   * Constants for test specs.
+   */
+  private static class Specs
+  {
+    static final String SERIALIZER_PATH = DATA_RESOURCE_ROOT + "/csv/serializer/serializer.json";
+    static final String INPUT_FORMAT_PATH = DATA_RESOURCE_ROOT + "/csv/input_format/input_format.json";
+    static final String PARSER_TYPE = AbstractStreamIndexingTest.INPUT_FORMAT;
+  }
+
+}
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
index 8cc8388ba47..f09eecea5b5 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
@@ -80,6 +80,7 @@ public abstract class AbstractStreamIndexingTest extends AbstractIndexerTest
 
   private static final String QUERIES_FILE = "/stream/queries/stream_index_queries.json";
   private static final String SUPERVISOR_SPEC_TEMPLATE_FILE = "supervisor_spec_template.json";
+  private static final String SUPERVISOR_WITH_CONCURRENT_LOCKS_SPEC_TEMPLATE_FILE = "supervisor_with_concurrent_locks_spec_template.json";
   private static final String SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_FILE = "supervisor_with_autoscaler_spec_template.json";
   private static final String SUPERVISOR_WITH_IDLE_BEHAVIOUR_ENABLED_SPEC_TEMPLATE_FILE =
       "supervisor_with_idle_behaviour_enabled_spec_template.json";
@@ -88,6 +89,8 @@ public abstract class AbstractStreamIndexingTest extends AbstractIndexerTest
       "supervisor_with_long_duration.json";
 
   protected static final String DATA_RESOURCE_ROOT = "/stream/data";
+  protected static final String SUPERVISOR_WITH_CONCURRENT_LOCKS_SPEC_TEMPLATE_PATH =
+      String.join("/", DATA_RESOURCE_ROOT, SUPERVISOR_WITH_CONCURRENT_LOCKS_SPEC_TEMPLATE_FILE);
   protected static final String SUPERVISOR_SPEC_TEMPLATE_PATH =
       String.join("/", DATA_RESOURCE_ROOT, SUPERVISOR_SPEC_TEMPLATE_FILE);
   protected static final String SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_PATH =
diff --git a/integration-tests/src/test/resources/stream/data/supervisor_with_concurrent_locks_spec_template.json b/integration-tests/src/test/resources/stream/data/supervisor_with_concurrent_locks_spec_template.json
new file mode 100644
index 00000000000..6adfc49da00
--- /dev/null
+++ b/integration-tests/src/test/resources/stream/data/supervisor_with_concurrent_locks_spec_template.json
@@ -0,0 +1,60 @@
+{
+  "type": "%%STREAM_TYPE%%",
+  "dataSchema": {
+    "dataSource": "%%DATASOURCE%%",
+    "parser": %%PARSER%%,
+    "timestampSpec": {
+      "column": "timestamp",
+      "format": "auto"
+    },
+    "dimensionsSpec": {
+      "dimensions": %%DIMENSIONS%%,
+      "dimensionExclusions": [],
+      "spatialDimensions": []
+    },
+    "metricsSpec": [
+      {
+        "type": "count",
+        "name": "count"
+      },
+      {
+        "type": "doubleSum",
+        "name": "added",
+        "fieldName": "added"
+      },
+      {
+        "type": "doubleSum",
+        "name": "deleted",
+        "fieldName": "deleted"
+      },
+      {
+        "type": "doubleSum",
+        "name": "delta",
+        "fieldName": "delta"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "MINUTE",
+      "queryGranularity": "NONE"
+    }
+  },
+  "tuningConfig": {
+    "type": "%%STREAM_TYPE%%",
+    "intermediatePersistPeriod": "PT30S",
+    "maxRowsPerSegment": 5000000,
+    "maxRowsInMemory": 500000
+  },
+  "ioConfig": {
+    "%%TOPIC_KEY%%": "%%TOPIC_VALUE%%",
+    "%%STREAM_PROPERTIES_KEY%%": %%STREAM_PROPERTIES_VALUE%%,
+    "taskCount": 2,
+    "replicas": 1,
+    "taskDuration": "PT120S",
+    "%%USE_EARLIEST_KEY%%": true,
+    "inputFormat" : %%INPUT_FORMAT%%
+  },
+  "context": {
+    "useConcurrentLocks": true
+  }
+}

From 786c959e9e3ea9a78f59917e3bab59d4141222af Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Tue, 3 Sep 2024 09:05:29 -0700
Subject: [PATCH 79/99] MSQ: Add limitHint to global-sort shuffles. (#16911)

* MSQ: Add limitHint to global-sort shuffles.

This allows pushing down limits into the SuperSorter.

* Test fixes.

* Add limitSpec to ScanQueryKit. Fix SuperSorter tracking.
---
 codestyle/spotbugs-exclude.xml                |  1 +
 .../apache/druid/msq/exec/RunWorkOrder.java   |  4 +-
 .../kernel/GlobalSortMaxCountShuffleSpec.java | 23 ++++-
 .../druid/msq/kernel/HashShuffleSpec.java     |  1 -
 .../kernel/LimitHintJsonIncludeFilter.java    | 38 ++++++++
 .../apache/druid/msq/kernel/ShuffleSpec.java  | 15 +++
 .../msq/querykit/ShuffleSpecFactories.java    | 23 +++--
 .../msq/querykit/groupby/GroupByQueryKit.java | 24 ++++-
 .../druid/msq/querykit/scan/ScanQueryKit.java | 24 +++--
 .../apache/druid/msq/exec/MSQSelectTest.java  |  4 +-
 .../indexing/report/MSQTaskReportTest.java    |  4 +-
 .../druid/msq/kernel/QueryDefinitionTest.java |  3 +-
 .../druid/msq/kernel/StageDefinitionTest.java |  6 +-
 .../MockQueryDefinitionBuilder.java           |  3 +-
 .../frame/processor/FrameChannelMerger.java   | 10 +-
 .../druid/frame/processor/SuperSorter.java    | 91 ++++++++++++++-----
 .../frame/processor/SuperSorterTest.java      | 82 +++++++++++++----
 17 files changed, 280 insertions(+), 76 deletions(-)
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/LimitHintJsonIncludeFilter.java

diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml
index 19da270192d..764ae7c7351 100644
--- a/codestyle/spotbugs-exclude.xml
+++ b/codestyle/spotbugs-exclude.xml
@@ -46,6 +46,7 @@
             <Or>
                 <Class name="org.apache.druid.jackson.DefaultTrueJsonIncludeFilter"/>
                 <Class name="org.apache.druid.java.util.common.StringEncodingDefaultUTF16LEJsonIncludeFilter"/>
+                <Class name="org.apache.druid.msq.kernel.LimitHintJsonIncludeFilter"/>
                 <Class name="org.apache.druid.query.scan.ScanQuery$ScanRowsLimitJsonIncludeFilter"/>
                 <Class name="org.apache.druid.query.scan.ScanQuery$ScanTimeOrderJsonIncludeFilter"/>
                 <Class name="org.apache.druid.query.scan.ScanQuery$BatchSizeJsonIncludeFilter"/>
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
index e48f1ef098a..a4d6a2180bd 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/RunWorkOrder.java
@@ -738,7 +738,7 @@ public class RunWorkOrder
                 makeSuperSorterIntermediateOutputChannelFactory(sorterTmpDir),
                 memoryParameters.getSuperSorterMaxActiveProcessors(),
                 memoryParameters.getSuperSorterMaxChannelsPerProcessor(),
-                -1,
+                stageDefinition.getShuffleSpec().limitHint(),
                 cancellationId,
                 counterTracker.sortProgress(),
                 removeNullBytes
@@ -871,7 +871,7 @@ public class RunWorkOrder
                         makeSuperSorterIntermediateOutputChannelFactory(sorterTmpDir),
                         1,
                         2,
-                        -1,
+                        ShuffleSpec.UNLIMITED,
                         cancellationId,
 
                         // Tracker is not actually tracked, since it doesn't quite fit into the way we report counters.
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/GlobalSortMaxCountShuffleSpec.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/GlobalSortMaxCountShuffleSpec.java
index e38fc778bb8..35576474ff9 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/GlobalSortMaxCountShuffleSpec.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/GlobalSortMaxCountShuffleSpec.java
@@ -43,17 +43,20 @@ public class GlobalSortMaxCountShuffleSpec implements GlobalSortShuffleSpec
   private final ClusterBy clusterBy;
   private final int maxPartitions;
   private final boolean aggregate;
+  private final long limitHint;
 
   @JsonCreator
   public GlobalSortMaxCountShuffleSpec(
       @JsonProperty("clusterBy") final ClusterBy clusterBy,
       @JsonProperty("partitions") final int maxPartitions,
-      @JsonProperty("aggregate") final boolean aggregate
+      @JsonProperty("aggregate") final boolean aggregate,
+      @JsonProperty("limitHint") final Long limitHint
   )
   {
     this.clusterBy = Preconditions.checkNotNull(clusterBy, "clusterBy");
     this.maxPartitions = maxPartitions;
     this.aggregate = aggregate;
+    this.limitHint = limitHint == null ? UNLIMITED : limitHint;
 
     if (maxPartitions < 1) {
       throw new IAE("Partition count must be at least 1");
@@ -133,6 +136,14 @@ public class GlobalSortMaxCountShuffleSpec implements GlobalSortShuffleSpec
     return maxPartitions;
   }
 
+  @Override
+  @JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = LimitHintJsonIncludeFilter.class)
+  @JsonProperty
+  public long limitHint()
+  {
+    return limitHint;
+  }
+
   @Override
   public boolean equals(Object o)
   {
@@ -145,22 +156,24 @@ public class GlobalSortMaxCountShuffleSpec implements GlobalSortShuffleSpec
     GlobalSortMaxCountShuffleSpec that = (GlobalSortMaxCountShuffleSpec) o;
     return maxPartitions == that.maxPartitions
            && aggregate == that.aggregate
-           && Objects.equals(clusterBy, that.clusterBy);
+           && Objects.equals(clusterBy, that.clusterBy)
+           && Objects.equals(limitHint, that.limitHint);
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(clusterBy, maxPartitions, aggregate);
+    return Objects.hash(clusterBy, maxPartitions, aggregate, limitHint);
   }
 
   @Override
   public String toString()
   {
-    return "MaxCountShuffleSpec{" +
+    return "GlobalSortMaxCountShuffleSpec{" +
            "clusterBy=" + clusterBy +
-           ", partitions=" + maxPartitions +
+           ", maxPartitions=" + maxPartitions +
            ", aggregate=" + aggregate +
+           ", limitHint=" + limitHint +
            '}';
   }
 }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/HashShuffleSpec.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/HashShuffleSpec.java
index fbc39fc672c..69e66fffe26 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/HashShuffleSpec.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/HashShuffleSpec.java
@@ -65,5 +65,4 @@ public class HashShuffleSpec implements ShuffleSpec
   {
     return numPartitions;
   }
-
 }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/LimitHintJsonIncludeFilter.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/LimitHintJsonIncludeFilter.java
new file mode 100644
index 00000000000..55572798e27
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/LimitHintJsonIncludeFilter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.kernel;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+/**
+ * {@link JsonInclude} filter for {@link ShuffleSpec#limitHint()}.
+ *
+ * This API works by "creative" use of equals. It requires warnings to be suppressed
+ * and also requires spotbugs exclusions (see spotbugs-exclude.xml).
+ */
+@SuppressWarnings({"EqualsAndHashcode", "EqualsHashCode"})
+public class LimitHintJsonIncludeFilter
+{
+  @Override
+  public boolean equals(Object obj)
+  {
+    return obj instanceof Long && (Long) obj == ShuffleSpec.UNLIMITED;
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/ShuffleSpec.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/ShuffleSpec.java
index 4b7971a7f78..97f3e6db547 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/ShuffleSpec.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/kernel/ShuffleSpec.java
@@ -37,6 +37,8 @@ import org.apache.druid.frame.key.ClusterBy;
 })
 public interface ShuffleSpec
 {
+  long UNLIMITED = -1;
+
   /**
    * The nature of this shuffle: hash vs. range based partitioning; whether the data are sorted or not.
    *
@@ -68,4 +70,17 @@ public interface ShuffleSpec
    * @throws IllegalStateException if kind is {@link ShuffleKind#GLOBAL_SORT} with more than one target partition
    */
   int partitionCount();
+
+  /**
+   * Limit that can be applied during shuffling. This is provided to enable performance optimizations.
+   *
+   * Implementations may apply this limit to each partition individually, or may apply it to the entire resultset
+   * (across all partitions). Either approach is valid, so downstream logic must handle either one.
+   *
+   * Implementations may also ignore this hint completely, or may apply a limit that is somewhat higher than this hint.
+   */
+  default long limitHint()
+  {
+    return UNLIMITED;
+  }
 }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/ShuffleSpecFactories.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/ShuffleSpecFactories.java
index d28439c0f8e..8f5770378c5 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/ShuffleSpecFactories.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/ShuffleSpecFactories.java
@@ -22,6 +22,7 @@ package org.apache.druid.msq.querykit;
 import org.apache.druid.msq.kernel.GlobalSortMaxCountShuffleSpec;
 import org.apache.druid.msq.kernel.GlobalSortTargetSizeShuffleSpec;
 import org.apache.druid.msq.kernel.MixShuffleSpec;
+import org.apache.druid.msq.kernel.ShuffleSpec;
 
 /**
  * Static factory methods for common implementations of {@link ShuffleSpecFactory}.
@@ -37,10 +38,21 @@ public class ShuffleSpecFactories
    * Factory that produces a single output partition, which may or may not be sorted.
    */
   public static ShuffleSpecFactory singlePartition()
+  {
+    return singlePartitionWithLimit(ShuffleSpec.UNLIMITED);
+  }
+
+  /**
+   * Factory that produces a single output partition, which may or may not be sorted.
+   *
+   * @param limitHint limit that can be applied during shuffling. May not actually be applied; this is just an
+   *                  optional optimization. See {@link ShuffleSpec#limitHint()}.
+   */
+  public static ShuffleSpecFactory singlePartitionWithLimit(final long limitHint)
   {
     return (clusterBy, aggregate) -> {
       if (clusterBy.sortable() && !clusterBy.isEmpty()) {
-        return new GlobalSortMaxCountShuffleSpec(clusterBy, 1, aggregate);
+        return new GlobalSortMaxCountShuffleSpec(clusterBy, 1, aggregate, limitHint);
       } else {
         return MixShuffleSpec.instance();
       }
@@ -52,7 +64,8 @@ public class ShuffleSpecFactories
    */
   public static ShuffleSpecFactory globalSortWithMaxPartitionCount(final int partitions)
   {
-    return (clusterBy, aggregate) -> new GlobalSortMaxCountShuffleSpec(clusterBy, partitions, aggregate);
+    return (clusterBy, aggregate) ->
+        new GlobalSortMaxCountShuffleSpec(clusterBy, partitions, aggregate, ShuffleSpec.UNLIMITED);
   }
 
   /**
@@ -61,10 +74,6 @@ public class ShuffleSpecFactories
   public static ShuffleSpecFactory getGlobalSortWithTargetSize(int targetSize)
   {
     return (clusterBy, aggregate) ->
-        new GlobalSortTargetSizeShuffleSpec(
-            clusterBy,
-            targetSize,
-            aggregate
-        );
+        new GlobalSortTargetSizeShuffleSpec(clusterBy, targetSize, aggregate);
   }
 }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByQueryKit.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByQueryKit.java
index 2bf77fd8d0c..7e4ebf5e7fa 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByQueryKit.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByQueryKit.java
@@ -112,13 +112,25 @@ public class GroupByQueryKit implements QueryKit<GroupByQuery>
     final ShuffleSpecFactory shuffleSpecFactoryPostAggregation;
     boolean partitionBoost;
 
+    // limitHint to use for the shuffle after the post-aggregation stage.
+    // Don't apply limitHint pre-aggregation, because results from pre-aggregation may not be fully grouped.
+    final long postAggregationLimitHint;
+
+    if (doLimitOrOffset) {
+      final DefaultLimitSpec limitSpec = (DefaultLimitSpec) queryToRun.getLimitSpec();
+      postAggregationLimitHint =
+          limitSpec.isLimited() ? limitSpec.getOffset() + limitSpec.getLimit() : ShuffleSpec.UNLIMITED;
+    } else {
+      postAggregationLimitHint = ShuffleSpec.UNLIMITED;
+    }
+
     if (intermediateClusterBy.isEmpty() && resultClusterByWithoutPartitionBoost.isEmpty()) {
       // Ignore shuffleSpecFactory, since we know only a single partition will come out, and we can save some effort.
       // This condition will be triggered when we don't have a grouping dimension, no partitioning granularity
       // (PARTITIONED BY ALL) and no ordering/clustering dimensions
       // For example: INSERT INTO foo SELECT COUNT(*) FROM bar PARTITIONED BY ALL
       shuffleSpecFactoryPreAggregation = ShuffleSpecFactories.singlePartition();
-      shuffleSpecFactoryPostAggregation = ShuffleSpecFactories.singlePartition();
+      shuffleSpecFactoryPostAggregation = ShuffleSpecFactories.singlePartitionWithLimit(postAggregationLimitHint);
       partitionBoost = false;
     } else if (doOrderBy) {
       // There can be a situation where intermediateClusterBy is empty, while the resultClusterBy is non-empty
@@ -130,9 +142,13 @@ public class GroupByQueryKit implements QueryKit<GroupByQuery>
       shuffleSpecFactoryPreAggregation = intermediateClusterBy.isEmpty()
                                          ? ShuffleSpecFactories.singlePartition()
                                          : ShuffleSpecFactories.globalSortWithMaxPartitionCount(maxWorkerCount);
-      shuffleSpecFactoryPostAggregation = doLimitOrOffset
-                                          ? ShuffleSpecFactories.singlePartition()
-                                          : resultShuffleSpecFactory;
+
+      if (doLimitOrOffset) {
+        shuffleSpecFactoryPostAggregation = ShuffleSpecFactories.singlePartitionWithLimit(postAggregationLimitHint);
+      } else {
+        shuffleSpecFactoryPostAggregation = resultShuffleSpecFactory;
+      }
+
       partitionBoost = true;
     } else {
       shuffleSpecFactoryPreAggregation = doLimitOrOffset
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryKit.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryKit.java
index 5bbf9c9cbb0..f4f50106e81 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryKit.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryKit.java
@@ -143,22 +143,32 @@ public class ScanQueryKit implements QueryKit<ScanQuery>
     );
 
     ShuffleSpec scanShuffleSpec;
-    if (!hasLimitOrOffset) {
-      // If there is no limit spec, apply the final shuffling here itself. This will ensure partition sizes etc are respected.
-      scanShuffleSpec = finalShuffleSpec;
-    } else {
+    if (hasLimitOrOffset) {
       // If there is a limit spec, check if there are any non-boost columns to sort in.
-      boolean requiresSort = clusterByColumns.stream()
-                                             .anyMatch(keyColumn -> !QueryKitUtils.PARTITION_BOOST_COLUMN.equals(keyColumn.columnName()));
+      boolean requiresSort =
+          clusterByColumns.stream()
+                          .anyMatch(keyColumn -> !QueryKitUtils.PARTITION_BOOST_COLUMN.equals(keyColumn.columnName()));
       if (requiresSort) {
         // If yes, do a sort into a single partition.
-        scanShuffleSpec = ShuffleSpecFactories.singlePartition().build(clusterBy, false);
+        final long limitHint;
+
+        if (queryToRun.isLimited()
+            && queryToRun.getScanRowsOffset() + queryToRun.getScanRowsLimit() > 0 /* overflow check */) {
+          limitHint = queryToRun.getScanRowsOffset() + queryToRun.getScanRowsLimit();
+        } else {
+          limitHint = ShuffleSpec.UNLIMITED;
+        }
+
+        scanShuffleSpec = ShuffleSpecFactories.singlePartitionWithLimit(limitHint).build(clusterBy, false);
       } else {
         // If the only clusterBy column is the boost column, we just use a mix shuffle to avoid unused shuffling.
         // Note that we still need the boost column to be present in the row signature, since the limit stage would
         // need it to be populated to do its own shuffling later.
         scanShuffleSpec = MixShuffleSpec.instance();
       }
+    } else {
+      // If there is no limit spec, apply the final shuffling here itself. This will ensure partition sizes etc are respected.
+      scanShuffleSpec = finalShuffleSpec;
     }
 
     queryDefBuilder.add(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
index b80b3844b50..305cdebf691 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
@@ -1847,7 +1847,7 @@ public class MSQSelectTest extends MSQTestBase
         )
         .setExpectedCountersForStageWorkerChannel(
             CounterSnapshotMatcher
-                .with().rows(5),
+                .with().rows(4),
             1, 0, "shuffle"
         )
         .setExpectedCountersForStageWorkerChannel(
@@ -1862,7 +1862,7 @@ public class MSQSelectTest extends MSQTestBase
         )
         .setExpectedCountersForStageWorkerChannel(
             CounterSnapshotMatcher
-                .with().rows(5),
+                .with().rows(4),
             2, 0, "input0"
         )
         .setExpectedCountersForStageWorkerChannel(
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java
index 4ab992aec09..3612aaf0db4 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java
@@ -37,6 +37,7 @@ import org.apache.druid.msq.indexing.error.MSQErrorReport;
 import org.apache.druid.msq.indexing.error.TooManyColumnsFault;
 import org.apache.druid.msq.kernel.GlobalSortMaxCountShuffleSpec;
 import org.apache.druid.msq.kernel.QueryDefinition;
+import org.apache.druid.msq.kernel.ShuffleSpec;
 import org.apache.druid.msq.kernel.StageDefinition;
 import org.apache.druid.msq.querykit.common.OffsetLimitFrameProcessorFactory;
 import org.apache.druid.segment.TestHelper;
@@ -69,7 +70,8 @@ public class MSQTaskReportTest
                       new GlobalSortMaxCountShuffleSpec(
                           new ClusterBy(ImmutableList.of(new KeyColumn("s", KeyOrder.ASCENDING)), 0),
                           2,
-                          false
+                          false,
+                          ShuffleSpec.UNLIMITED
                       )
                   )
                   .maxWorkerCount(3)
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/QueryDefinitionTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/QueryDefinitionTest.java
index 857584127a9..584e8147fec 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/QueryDefinitionTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/QueryDefinitionTest.java
@@ -51,7 +51,8 @@ public class QueryDefinitionTest
                         new GlobalSortMaxCountShuffleSpec(
                             new ClusterBy(ImmutableList.of(new KeyColumn("s", KeyOrder.ASCENDING)), 0),
                             2,
-                            false
+                            false,
+                            ShuffleSpec.UNLIMITED
                         )
                     )
                     .maxWorkerCount(3)
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/StageDefinitionTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/StageDefinitionTest.java
index 20110b63329..7805154799d 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/StageDefinitionTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/StageDefinitionTest.java
@@ -74,7 +74,8 @@ public class StageDefinitionTest
         new GlobalSortMaxCountShuffleSpec(
             new ClusterBy(ImmutableList.of(new KeyColumn("test", KeyOrder.ASCENDING)), 0),
             2,
-            false
+            false,
+            ShuffleSpec.UNLIMITED
         ),
         1,
         false
@@ -95,7 +96,8 @@ public class StageDefinitionTest
         new GlobalSortMaxCountShuffleSpec(
             new ClusterBy(ImmutableList.of(new KeyColumn("test", KeyOrder.ASCENDING)), 0),
             1,
-            false
+            false,
+            ShuffleSpec.UNLIMITED
         ),
         1,
         false
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/controller/MockQueryDefinitionBuilder.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/controller/MockQueryDefinitionBuilder.java
index 6ac399f70e4..4d3880fefbe 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/controller/MockQueryDefinitionBuilder.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/kernel/controller/MockQueryDefinitionBuilder.java
@@ -139,7 +139,8 @@ public class MockQueryDefinitionBuilder
                   0
               ),
               MAX_NUM_PARTITIONS,
-              false
+              false,
+              ShuffleSpec.UNLIMITED
           );
           break;
 
diff --git a/processing/src/main/java/org/apache/druid/frame/processor/FrameChannelMerger.java b/processing/src/main/java/org/apache/druid/frame/processor/FrameChannelMerger.java
index df343be2728..e806f98b926 100644
--- a/processing/src/main/java/org/apache/druid/frame/processor/FrameChannelMerger.java
+++ b/processing/src/main/java/org/apache/druid/frame/processor/FrameChannelMerger.java
@@ -177,14 +177,20 @@ public class FrameChannelMerger implements FrameProcessor<Long>
       return ReturnOrAwait.awaitAll(awaitSet);
     }
 
+    // Check finished() after populateCurrentFramesAndTournamentTree().
     if (finished()) {
-      // Done!
       return ReturnOrAwait.returnObject(rowsOutput);
     }
 
     // Generate one output frame and stop for now.
     outputChannel.write(nextFrame());
-    return ReturnOrAwait.runAgain();
+
+    // Check finished() after nextFrame().
+    if (finished()) {
+      return ReturnOrAwait.returnObject(rowsOutput);
+    } else {
+      return ReturnOrAwait.runAgain();
+    }
   }
 
   private FrameWithPartition nextFrame()
diff --git a/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java b/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
index d2b6934d292..e30f2e77b02 100644
--- a/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
+++ b/processing/src/main/java/org/apache/druid/frame/processor/SuperSorter.java
@@ -123,6 +123,7 @@ public class SuperSorter
 {
   private static final Logger log = new Logger(SuperSorter.class);
 
+  public static final long UNLIMITED = -1;
   public static final int UNKNOWN_LEVEL = -1;
   public static final long UNKNOWN_TOTAL = -1;
 
@@ -136,10 +137,8 @@ public class SuperSorter
   private final OutputChannelFactory intermediateOutputChannelFactory;
   private final int maxChannelsPerMerger;
   private final int maxActiveProcessors;
-  private final long rowLimit;
   private final String cancellationId;
   private final boolean removeNullBytes;
-
   private final Object runWorkersLock = new Object();
 
   @GuardedBy("runWorkersLock")
@@ -184,6 +183,9 @@ public class SuperSorter
   @GuardedBy("runWorkersLock")
   SuperSorterProgressTracker superSorterProgressTracker;
 
+  @GuardedBy("runWorkersLock")
+  private long rowLimit;
+
   /**
    * See {@link #setNoWorkRunnable}.
    */
@@ -212,9 +214,9 @@ public class SuperSorter
    * @param maxChannelsPerMerger             maximum number of channels to merge at once, for regular mergers
    *                                         (does not apply to direct mergers; see
    *                                         {@link #getMaxInputBufferFramesForDirectMerging()})
-   * @param rowLimit                         limit to apply during sorting. The limit is merely advisory: the actual number
-   *                                         of rows returned may be larger than the limit. The limit is applied across
-   *                                         all partitions, not to each partition individually.
+   * @param rowLimit                         limit to apply during sorting. The limit is applied across all partitions,
+   *                                         not to each partition individually. Use {@link #UNLIMITED} if there is
+   *                                         no limit.
    * @param cancellationId                   cancellation id to use when running processors in the provided
    *                                         {@link FrameProcessorExecutor}.
    * @param superSorterProgressTracker       progress tracker
@@ -262,6 +264,10 @@ public class SuperSorter
     if (maxChannelsPerMerger < 2) {
       throw new IAE("maxChannelsPerMerger[%d] < 2", maxChannelsPerMerger);
     }
+
+    if (rowLimit != UNLIMITED && rowLimit <= 0) {
+      throw new IAE("rowLimit[%d] must be positive", rowLimit);
+    }
   }
 
   /**
@@ -385,29 +391,41 @@ public class SuperSorter
   @GuardedBy("runWorkersLock")
   private void setAllDoneIfPossible()
   {
-    if (totalInputFrames == 0 && outputPartitionsFuture.isDone()) {
-      // No input data -- generate empty output channels.
-      final ClusterByPartitions partitions = getOutputPartitions();
-      final List<OutputChannel> channels = new ArrayList<>(partitions.size());
+    try {
+      if (totalInputFrames == 0 && outputPartitionsFuture.isDone()) {
+        // No input data -- generate empty output channels.
+        final ClusterByPartitions partitions = getOutputPartitions();
+        final List<OutputChannel> channels = new ArrayList<>(partitions.size());
 
-      for (int partitionNum = 0; partitionNum < partitions.size(); partitionNum++) {
-        channels.add(outputChannelFactory.openNilChannel(partitionNum));
-      }
+        for (int partitionNum = 0; partitionNum < partitions.size(); partitionNum++) {
+          channels.add(outputChannelFactory.openNilChannel(partitionNum));
+        }
 
-      // OK to use wrap, not wrapReadOnly, because nil channels are already read-only.
-      allDone.set(OutputChannels.wrap(channels));
-    } else if (totalMergingLevels != UNKNOWN_LEVEL
-               && outputsReadyByLevel.containsKey(totalMergingLevels - 1)
-               && (outputsReadyByLevel.get(totalMergingLevels - 1).size() ==
-                   getTotalMergersInLevel(totalMergingLevels - 1))) {
-      // We're done!!
-      try {
+        // OK to use wrap, not wrapReadOnly, because nil channels are already read-only.
+        allDone.set(OutputChannels.wrap(channels));
+      } else if (rowLimit == 0 && activeProcessors == 0) {
+        // We had a row limit, and got it all the way down to zero.
+        // Generate empty output channels for any partitions that we haven't written yet.
+        for (int partitionNum = 0; partitionNum < outputChannels.size(); partitionNum++) {
+          if (outputChannels.get(partitionNum) == null) {
+            outputChannels.set(partitionNum, outputChannelFactory.openNilChannel(partitionNum));
+            superSorterProgressTracker.addMergedBatchesForLevel(totalMergingLevels - 1, 1);
+          }
+        }
+
+        // OK to use wrap, not wrapReadOnly, because all channels in this list are already read-only.
+        allDone.set(OutputChannels.wrap(outputChannels));
+      } else if (totalMergingLevels != UNKNOWN_LEVEL
+                 && outputsReadyByLevel.containsKey(totalMergingLevels - 1)
+                 && (outputsReadyByLevel.get(totalMergingLevels - 1).size() ==
+                     getTotalMergersInLevel(totalMergingLevels - 1))) {
+        // We're done!!
         // OK to use wrap, not wrapReadOnly, because all channels in this list are already read-only.
         allDone.set(OutputChannels.wrap(outputChannels));
       }
-      catch (Throwable e) {
-        allDone.setException(e);
-      }
+    }
+    catch (Throwable e) {
+      allDone.setException(e);
     }
   }
 
@@ -463,6 +481,11 @@ public class SuperSorter
       return false;
     }
 
+    if (isLimited() && (rowLimit == 0 || activeProcessors > 0)) {
+      // Run final-layer mergers one at a time, to ensure limit is applied across the entire dataset.
+      return false;
+    }
+
     final List<ReadableFrameChannel> in = new ArrayList<>();
 
     for (final Frame frame : inputBuffer) {
@@ -617,6 +640,11 @@ public class SuperSorter
       return false;
     }
 
+    if (isLimited() && (rowLimit == 0 || activeProcessors > 0)) {
+      // Run final-layer mergers one at a time, to ensure limit is applied across the entire dataset.
+      return false;
+    }
+
     final int inLevel = totalMergingLevels - 2;
     final int outLevel = inLevel + 1;
     final LongSortedSet inputsReady = outputsReadyByLevel.get(inLevel);
@@ -719,11 +747,20 @@ public class SuperSorter
               rowLimit
           );
 
-      runWorker(worker, ignored1 -> {
+      runWorker(worker, outputRows -> {
         synchronized (runWorkersLock) {
           outputsReadyByLevel.computeIfAbsent(level, ignored2 -> new LongRBTreeSet())
                              .add(rank);
           superSorterProgressTracker.addMergedBatchesForLevel(level, 1);
+
+          if (isLimited() && totalMergingLevels != UNKNOWN_LEVEL && level == totalMergingLevels - 1) {
+            rowLimit -= outputRows;
+
+            if (rowLimit < 0) {
+              throw DruidException.defensive("rowLimit[%d] below zero after outputRows[%d]", rowLimit, outputRows);
+            }
+          }
+
           for (PartitionedReadableFrameChannel partitionedReadableFrameChannel : partitionedReadableChannelsToClose) {
             try {
               partitionedReadableFrameChannel.close();
@@ -984,6 +1021,12 @@ public class SuperSorter
     return maxChannelsPerMerger * maxActiveProcessors;
   }
 
+  @GuardedBy("runWorkersLock")
+  private boolean isLimited()
+  {
+    return rowLimit != UNLIMITED;
+  }
+
   /**
    * Returns a string encapsulating the current state of this object.
    */
diff --git a/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java b/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
index 36644f6d771..ab5b1c8197f 100644
--- a/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
@@ -21,6 +21,7 @@ package org.apache.druid.frame.processor;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import com.google.common.primitives.Ints;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.SettableFuture;
@@ -58,6 +59,8 @@ import org.apache.druid.segment.TestIndex;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -131,7 +134,7 @@ public class SuperSorterTest
           new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
           2,
           2,
-          -1,
+          SuperSorter.UNLIMITED,
           null,
           superSorterProgressTracker,
           false
@@ -181,6 +184,40 @@ public class SuperSorterTest
       Assert.assertEquals(1.0, superSorterProgressTracker.snapshot().getProgressDigest(), 0.0f);
       channel.close();
     }
+
+    @Test
+    public void testLimitHint() throws Exception
+    {
+      final BlockingQueueFrameChannel inputChannel = BlockingQueueFrameChannel.minimal();
+      inputChannel.writable().close();
+
+      final SuperSorterProgressTracker superSorterProgressTracker = new SuperSorterProgressTracker();
+
+      final File tempFolder = temporaryFolder.newFolder();
+      final SuperSorter superSorter = new SuperSorter(
+          Collections.singletonList(inputChannel.readable()),
+          FrameReader.create(RowSignature.empty()),
+          Collections.emptyList(),
+          Futures.immediateFuture(ClusterByPartitions.oneUniversalPartition()),
+          exec,
+          new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
+          new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
+          2,
+          2,
+          3,
+          null,
+          superSorterProgressTracker,
+          false
+      );
+
+      final OutputChannels channels = superSorter.run().get();
+      Assert.assertEquals(1, channels.getAllChannels().size());
+
+      final ReadableFrameChannel channel = Iterables.getOnlyElement(channels.getAllChannels()).getReadableChannel();
+      Assert.assertTrue(channel.isFinished());
+      Assert.assertEquals(1.0, superSorterProgressTracker.snapshot().getProgressDigest(), 0.0f);
+      channel.close();
+    }
   }
 
   /**
@@ -201,6 +238,7 @@ public class SuperSorterTest
     private final int numThreads;
     private final boolean isComposedStorage;
     private final boolean partitionsDeferred;
+    private final long limitHint;
 
     private StorageAdapter adapter;
     private RowSignature signature;
@@ -216,7 +254,8 @@ public class SuperSorterTest
         int maxChannelsPerProcessor,
         int numThreads,
         boolean isComposedStorage,
-        boolean partitionsDeferred
+        boolean partitionsDeferred,
+        long limitHint
     )
     {
       this.maxRowsPerFrame = maxRowsPerFrame;
@@ -227,6 +266,7 @@ public class SuperSorterTest
       this.numThreads = numThreads;
       this.isComposedStorage = isComposedStorage;
       this.partitionsDeferred = partitionsDeferred;
+      this.limitHint = limitHint;
     }
 
     @Parameterized.Parameters(
@@ -237,7 +277,8 @@ public class SuperSorterTest
                + "maxChannelsPerProcessor= {4}, "
                + "numThreads = {5}, "
                + "isComposedStorage = {6}, "
-               + "partitionsDeferred = {7}"
+               + "partitionsDeferred = {7}, "
+               + "limitHint = {8}"
     )
     public static Iterable<Object[]> constructorFeeder()
     {
@@ -251,18 +292,21 @@ public class SuperSorterTest
                 for (int numThreads : new int[]{1, 3}) {
                   for (boolean isComposedStorage : new boolean[]{true, false}) {
                     for (boolean partitionsDeferred : new boolean[]{true, false}) {
-                      constructors.add(
-                          new Object[]{
-                              maxRowsPerFrame,
-                              maxBytesPerFrame,
-                              numChannels,
-                              maxActiveProcessors,
-                              maxChannelsPerProcessor,
-                              numThreads,
-                              isComposedStorage,
-                              partitionsDeferred
-                          }
-                      );
+                      for (long limitHint : new long[]{SuperSorter.UNLIMITED, 3, 1_000}) {
+                        constructors.add(
+                            new Object[]{
+                                maxRowsPerFrame,
+                                maxBytesPerFrame,
+                                numChannels,
+                                maxActiveProcessors,
+                                maxChannelsPerProcessor,
+                                numThreads,
+                                isComposedStorage,
+                                partitionsDeferred,
+                                limitHint
+                            }
+                        );
+                      }
                     }
                   }
                 }
@@ -352,7 +396,7 @@ public class SuperSorterTest
           outputChannelFactory,
           maxActiveProcessors,
           maxChannelsPerProcessor,
-          -1,
+          limitHint,
           null,
           superSorterProgressTracker,
           false
@@ -415,6 +459,10 @@ public class SuperSorterTest
         );
       }
 
+      if (limitHint != SuperSorter.UNLIMITED) {
+        MatcherAssert.assertThat(readRows.size(), Matchers.greaterThanOrEqualTo(Ints.checkedCast(limitHint)));
+      }
+
       final Sequence<List<Object>> expectedRows = Sequences.sort(
           FrameTestUtil.readRowsFromAdapter(adapter, signature, true),
           Comparator.comparing(
@@ -429,7 +477,7 @@ public class SuperSorterTest
               },
               keyComparator
           )
-      );
+      ).limit(limitHint == SuperSorter.UNLIMITED ? Long.MAX_VALUE : readRows.size());
 
       FrameTestUtil.assertRowsEqual(expectedRows, Sequences.simple(readRows));
     }

From 2ef936be40bcb7a9295ea9ee95e79c2c64921f75 Mon Sep 17 00:00:00 2001
From: Hardik Bajaj <58038410+hardikbajaj@users.noreply.github.com>
Date: Wed, 4 Sep 2024 00:25:09 +0530
Subject: [PATCH 80/99] Update Documentation on meregeBuffer/pendingRequests
 for Real-time nodes (#16992)

#15025 adds mergeBuffer/pendingRequests metric in QueryCountStatsMonitor. Since real-time nodes also use the same merge buffers for queries and have QueryCountStatsMonitor , the documentation is being updated to include this metric.
---
 docs/operations/metrics.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md
index 83cc9550690..dc2ee952813 100644
--- a/docs/operations/metrics.md
+++ b/docs/operations/metrics.md
@@ -119,6 +119,7 @@ Most metric values reset each emission period, as specified in `druid.monitoring
 |`query/failed/count`|Number of failed queries.|This metric is only available if the `QueryCountStatsMonitor` module is included.||
 |`query/interrupted/count`|Number of queries interrupted due to cancellation.|This metric is only available if the `QueryCountStatsMonitor` module is included.||
 |`query/timeout/count`|Number of timed out queries.|This metric is only available if the `QueryCountStatsMonitor` module is included.||
+|`mergeBuffer/pendingRequests`|Number of requests waiting to acquire a batch of buffers from the merge buffer pool.|This metric is only available if the `QueryCountStatsMonitor` module is included.||
 
 ### Jetty
 

From 57c4b552d98b9c59f89f2307e1146b336771657c Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Tue, 3 Sep 2024 13:14:59 -0700
Subject: [PATCH 81/99] Fix logical merge conflict in SuperSorterTest. (#16993)

Logical merge conflict between #16911 and #16914.
---
 .../java/org/apache/druid/frame/processor/SuperSorterTest.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java b/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
index ab5b1c8197f..b40fde04a9a 100644
--- a/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/processor/SuperSorterTest.java
@@ -200,6 +200,7 @@ public class SuperSorterTest
           Collections.emptyList(),
           Futures.immediateFuture(ClusterByPartitions.oneUniversalPartition()),
           exec,
+          FrameProcessorDecorator.NONE,
           new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
           new FileOutputChannelFactory(tempFolder, FRAME_SIZE, null),
           2,

From 57bf053dc981e68202c7bc1037a7ec09f31210d7 Mon Sep 17 00:00:00 2001
From: Clint Wylie <cwylie@apache.org>
Date: Tue, 3 Sep 2024 20:04:30 -0700
Subject: [PATCH 82/99] remove compiler warnings about unqualified calls to
 yield() (#16995)

---
 .../org/apache/druid/common/guava/CombiningSequenceTest.java    | 2 +-
 .../java/org/apache/druid/common/guava/ComplexSequenceTest.java | 2 +-
 .../apache/druid/java/util/common/guava/ConcatSequenceTest.java | 2 +-
 .../druid/java/util/common/guava/LimitedSequenceTest.java       | 2 +-
 .../apache/druid/java/util/common/guava/SequenceTestHelper.java | 2 +-
 .../druid/java/util/common/guava/SkippingSequenceTest.java      | 2 +-
 .../apache/druid/query/aggregation/AggregationTestHelper.java   | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/processing/src/test/java/org/apache/druid/common/guava/CombiningSequenceTest.java b/processing/src/test/java/org/apache/druid/common/guava/CombiningSequenceTest.java
index 7c97e364828..437c3825170 100644
--- a/processing/src/test/java/org/apache/druid/common/guava/CombiningSequenceTest.java
+++ b/processing/src/test/java/org/apache/druid/common/guava/CombiningSequenceTest.java
@@ -294,7 +294,7 @@ public class CombiningSequenceTest
           {
             count++;
             if (count % yieldEvery == 0) {
-              yield();
+              this.yield();
             }
             return rhs;
           }
diff --git a/processing/src/test/java/org/apache/druid/common/guava/ComplexSequenceTest.java b/processing/src/test/java/org/apache/druid/common/guava/ComplexSequenceTest.java
index ba5504f25cf..bf1099ccc0f 100644
--- a/processing/src/test/java/org/apache/druid/common/guava/ComplexSequenceTest.java
+++ b/processing/src/test/java/org/apache/druid/common/guava/ComplexSequenceTest.java
@@ -71,7 +71,7 @@ public class ComplexSequenceTest
           @Override
           public Integer accumulate(Integer accumulated, Integer in)
           {
-            yield();
+            this.yield();
             return in;
           }
         }
diff --git a/processing/src/test/java/org/apache/druid/java/util/common/guava/ConcatSequenceTest.java b/processing/src/test/java/org/apache/druid/java/util/common/guava/ConcatSequenceTest.java
index cf5cabe99a8..477357e0792 100644
--- a/processing/src/test/java/org/apache/druid/java/util/common/guava/ConcatSequenceTest.java
+++ b/processing/src/test/java/org/apache/druid/java/util/common/guava/ConcatSequenceTest.java
@@ -263,7 +263,7 @@ public class ConcatSequenceTest
           @Override
           public Integer accumulate(Integer accumulated, Integer in)
           {
-            yield();
+            this.yield();
             return in;
           }
         }
diff --git a/processing/src/test/java/org/apache/druid/java/util/common/guava/LimitedSequenceTest.java b/processing/src/test/java/org/apache/druid/java/util/common/guava/LimitedSequenceTest.java
index 9ea8b75aa31..d26a950bdd7 100644
--- a/processing/src/test/java/org/apache/druid/java/util/common/guava/LimitedSequenceTest.java
+++ b/processing/src/test/java/org/apache/druid/java/util/common/guava/LimitedSequenceTest.java
@@ -104,7 +104,7 @@ public class LimitedSequenceTest
               public OutType accumulate(OutType accumulated, Integer in)
               {
                 final OutType retVal = super.accumulate(accumulated, in);
-                yield();
+                this.yield();
                 return retVal;
               }
             }
diff --git a/processing/src/test/java/org/apache/druid/java/util/common/guava/SequenceTestHelper.java b/processing/src/test/java/org/apache/druid/java/util/common/guava/SequenceTestHelper.java
index f8a3798954f..afbb960e00a 100644
--- a/processing/src/test/java/org/apache/druid/java/util/common/guava/SequenceTestHelper.java
+++ b/processing/src/test/java/org/apache/druid/java/util/common/guava/SequenceTestHelper.java
@@ -65,7 +65,7 @@ public class SequenceTestHelper
           {
             if (++count >= numToTake) {
               count = 0;
-              yield();
+              this.yield();
             }
 
             Assert.assertEquals(prefix, valsIter.next(), in);
diff --git a/processing/src/test/java/org/apache/druid/java/util/common/guava/SkippingSequenceTest.java b/processing/src/test/java/org/apache/druid/java/util/common/guava/SkippingSequenceTest.java
index 0253794fd03..2b299d0b79b 100644
--- a/processing/src/test/java/org/apache/druid/java/util/common/guava/SkippingSequenceTest.java
+++ b/processing/src/test/java/org/apache/druid/java/util/common/guava/SkippingSequenceTest.java
@@ -110,7 +110,7 @@ public class SkippingSequenceTest
               public OutType accumulate(OutType accumulated, Integer in)
               {
                 final OutType retVal = super.accumulate(accumulated, in);
-                yield();
+                this.yield();
                 return retVal;
               }
             }
diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java
index e21b185d5cd..c1d643d519e 100644
--- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java
+++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java
@@ -757,7 +757,7 @@ public class AggregationTestHelper implements Closeable
                 @Override
                 public Object accumulate(Object accumulated, Object in)
                 {
-                  yield();
+                  this.yield();
                   return in;
                 }
               }

From 76b8c20f4d79db65e149c9855ea924002825c0d6 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Tue, 3 Sep 2024 20:04:44 -0700
Subject: [PATCH 83/99] Create fewer temporary maps when querying sys.segments.
 (#16981)

Eliminates two map creations (availableSegmentMetadata, partialSegmentDataMap).
The segmentsAlreadySeen set remains.
---
 .../AbstractSegmentMetadataCache.java         | 30 ++++++--
 .../CoordinatorSegmentMetadataCache.java      | 45 ++++++------
 .../sql/calcite/schema/SystemSchema.java      | 72 ++++++++-----------
 3 files changed, 77 insertions(+), 70 deletions(-)

diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
index 06ce009c8b2..a2d5999a0e1 100644
--- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
@@ -67,6 +67,7 @@ import java.io.IOException;
 import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -413,13 +414,28 @@ public abstract class AbstractSegmentMetadataCache<T extends DataSourceInformati
    */
   public Map<SegmentId, AvailableSegmentMetadata> getSegmentMetadataSnapshot()
   {
-    final Map<SegmentId, AvailableSegmentMetadata> segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments);
-    for (ConcurrentSkipListMap<SegmentId, AvailableSegmentMetadata> val : segmentMetadataInfo.values()) {
-      segmentMetadata.putAll(val);
+    final Map<SegmentId, AvailableSegmentMetadata> segmentMetadata = Maps.newHashMapWithExpectedSize(getTotalSegments());
+    final Iterator<AvailableSegmentMetadata> it = iterateSegmentMetadata();
+    while (it.hasNext()) {
+      final AvailableSegmentMetadata availableSegmentMetadata = it.next();
+      segmentMetadata.put(availableSegmentMetadata.getSegment().getId(), availableSegmentMetadata);
     }
     return segmentMetadata;
   }
 
+  /**
+   * Get metadata for all the cached segments, which includes information like RowSignature, realtime & numRows etc.
+   * This is a lower-overhead method than {@link #getSegmentMetadataSnapshot()}.
+   *
+   * @return iterator of metadata.
+   */
+  public Iterator<AvailableSegmentMetadata> iterateSegmentMetadata()
+  {
+    return FluentIterable.from(segmentMetadataInfo.values())
+                         .transformAndConcat(Map::values)
+                         .iterator();
+  }
+
   /**
    * Get metadata for the specified segment, which includes information like RowSignature, realtime & numRows.
    *
@@ -431,10 +447,14 @@ public abstract class AbstractSegmentMetadataCache<T extends DataSourceInformati
   @Nullable
   public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId)
   {
-    if (!segmentMetadataInfo.containsKey(datasource)) {
+    final ConcurrentSkipListMap<SegmentId, AvailableSegmentMetadata> dataSourceMap =
+        segmentMetadataInfo.get(datasource);
+
+    if (dataSourceMap == null) {
       return null;
+    } else {
+      return dataSourceMap.get(segmentId);
     }
-    return segmentMetadataInfo.get(datasource).get(segmentId);
   }
 
   /**
diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
index 0c03f7af73c..49254c1da5c 100644
--- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
@@ -21,8 +21,8 @@ package org.apache.druid.segment.metadata;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Supplier;
+import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.inject.Inject;
 import org.apache.druid.client.CoordinatorServerView;
@@ -64,6 +64,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -357,27 +358,29 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
   }
 
   @Override
-  public Map<SegmentId, AvailableSegmentMetadata> getSegmentMetadataSnapshot()
+  public Iterator<AvailableSegmentMetadata> iterateSegmentMetadata()
   {
-    final Map<SegmentId, AvailableSegmentMetadata> segmentMetadata = Maps.newHashMapWithExpectedSize(getTotalSegments());
-    for (ConcurrentSkipListMap<SegmentId, AvailableSegmentMetadata> val : segmentMetadataInfo.values()) {
-      for (Map.Entry<SegmentId, AvailableSegmentMetadata> entry : val.entrySet()) {
-        Optional<SchemaPayloadPlus> metadata = segmentSchemaCache.getSchemaForSegment(entry.getKey());
-        AvailableSegmentMetadata availableSegmentMetadata = entry.getValue();
-        if (metadata.isPresent()) {
-          availableSegmentMetadata = AvailableSegmentMetadata.from(entry.getValue())
-                                           .withRowSignature(metadata.get().getSchemaPayload().getRowSignature())
-                                           .withNumRows(metadata.get().getNumRows())
-                                           .build();
-        } else {
-          // mark it for refresh, however, this case shouldn't arise by design
-          markSegmentAsNeedRefresh(entry.getKey());
-          log.debug("SchemaMetadata for segmentId [%s] is absent.", entry.getKey());
-        }
-        segmentMetadata.put(entry.getKey(), availableSegmentMetadata);
-      }
-    }
-    return segmentMetadata;
+    return FluentIterable
+        .from(segmentMetadataInfo.values())
+        .transformAndConcat(Map::values)
+        .transform(
+            availableSegmentMetadata -> {
+              final SegmentId segmentId = availableSegmentMetadata.getSegment().getId();
+              final Optional<SchemaPayloadPlus> metadata = segmentSchemaCache.getSchemaForSegment(segmentId);
+              if (metadata.isPresent()) {
+                return AvailableSegmentMetadata.from(availableSegmentMetadata)
+                                               .withRowSignature(metadata.get().getSchemaPayload().getRowSignature())
+                                               .withNumRows(metadata.get().getNumRows())
+                                               .build();
+              } else {
+                // mark it for refresh, however, this case shouldn't arise by design
+                markSegmentAsNeedRefresh(segmentId);
+                log.debug("SchemaMetadata for segmentId[%s] is absent.", segmentId);
+                return availableSegmentMetadata;
+              }
+            }
+        )
+        .iterator();
   }
 
   @Nullable
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
index 695b34b2772..0af8c19ca06 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
@@ -28,7 +28,6 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Futures;
 import com.google.inject.Inject;
@@ -100,7 +99,6 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -304,38 +302,29 @@ public class SystemSchema extends AbstractSchema
     )
     {
       // get available segments from druidSchema
-      final Map<SegmentId, AvailableSegmentMetadata> availableSegmentMetadata =
-          druidSchema.cache().getSegmentMetadataSnapshot();
-      final Iterator<Entry<SegmentId, AvailableSegmentMetadata>> availableSegmentEntries =
-          availableSegmentMetadata.entrySet().iterator();
+      final BrokerSegmentMetadataCache availableMetadataCache = druidSchema.cache();
 
-      // in memory map to store segment data from available segments
-      final Map<SegmentId, PartialSegmentData> partialSegmentDataMap =
-          Maps.newHashMapWithExpectedSize(druidSchema.cache().getTotalSegments());
-      for (AvailableSegmentMetadata h : availableSegmentMetadata.values()) {
-        PartialSegmentData partialSegmentData =
-            new PartialSegmentData(IS_AVAILABLE_TRUE, h.isRealtime(), h.getNumReplicas(), h.getNumRows());
-        partialSegmentDataMap.put(h.getSegment().getId(), partialSegmentData);
-      }
+      // Keep track of which segments we emitted from the publishedSegments iterator, so we don't emit them again
+      // from the availableSegments iterator.
+      final Set<SegmentId> segmentsAlreadySeen =
+          Sets.newHashSetWithExpectedSize(availableMetadataCache.getTotalSegments());
 
       // Get segments from metadata segment cache (if enabled in SQL planner config), else directly from
       // Coordinator. This may include both published and realtime segments.
       final Iterator<SegmentStatusInCluster> metadataStoreSegments = metadataView.getSegments();
-
-      final Set<SegmentId> segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(druidSchema.cache().getTotalSegments());
-
       final FluentIterable<Object[]> publishedSegments = FluentIterable
           .from(() -> getAuthorizedPublishedSegments(metadataStoreSegments, root))
           .transform(val -> {
             final DataSegment segment = val.getDataSegment();
+            final AvailableSegmentMetadata availableSegmentMetadata =
+                availableMetadataCache.getAvailableSegmentMetadata(segment.getDataSource(), segment.getId());
             segmentsAlreadySeen.add(segment.getId());
-            final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(segment.getId());
             long numReplicas = 0L, numRows = 0L, isRealtime, isAvailable = 0L;
 
-            if (partialSegmentData != null) {
-              numReplicas = partialSegmentData.getNumReplicas();
-              isAvailable = partialSegmentData.isAvailable();
-              numRows = partialSegmentData.getNumRows();
+            if (availableSegmentMetadata != null) {
+              numReplicas = availableSegmentMetadata.getNumReplicas();
+              isAvailable = availableSegmentMetadata.getNumReplicas() > 0 ? IS_AVAILABLE_TRUE : IS_ACTIVE_FALSE;
+              numRows = availableSegmentMetadata.getNumRows();
             }
 
             // If druid.centralizedDatasourceSchema.enabled is set on the Coordinator, SegmentMetadataCache on the
@@ -383,34 +372,29 @@ public class SystemSchema extends AbstractSchema
       // If druid.centralizedDatasourceSchema.enabled is set on the Coordinator, all the segments in this loop
       // would be covered in the previous iteration since Coordinator would return realtime segments as well.
       final FluentIterable<Object[]> availableSegments = FluentIterable
-          .from(() -> getAuthorizedAvailableSegments(
-              availableSegmentEntries,
-              root
-          ))
+          .from(() -> getAuthorizedAvailableSegments(availableMetadataCache.iterateSegmentMetadata(), root))
           .transform(val -> {
-            if (segmentsAlreadySeen.contains(val.getKey())) {
+            final DataSegment segment = val.getSegment();
+            if (segmentsAlreadySeen.contains(segment.getId())) {
               return null;
             }
-            final DataSegment segment = val.getValue().getSegment();
-            final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(val.getKey());
-            final long numReplicas = partialSegmentData == null ? 0L : partialSegmentData.getNumReplicas();
             return new Object[]{
-                val.getKey(),
-                val.getKey().getDataSource(),
-                val.getKey().getInterval().getStart(),
-                val.getKey().getInterval().getEnd(),
+                segment.getId(),
+                segment.getDataSource(),
+                segment.getInterval().getStart(),
+                segment.getInterval().getEnd(),
                 segment.getSize(),
-                val.getKey().getVersion(),
+                segment.getVersion(),
                 (long) segment.getShardSpec().getPartitionNum(),
-                numReplicas,
-                val.getValue().getNumRows(),
+                val.getNumReplicas(),
+                val.getNumRows(),
                 // is_active is true for unpublished segments iff they are realtime
-                val.getValue().isRealtime() /* is_active */,
+                val.isRealtime() /* is_active */,
                 // is_published is false for unpublished segments
                 IS_PUBLISHED_FALSE,
                 // is_available is assumed to be always true for segments announced by historicals or realtime tasks
                 IS_AVAILABLE_TRUE,
-                val.getValue().isRealtime(),
+                val.isRealtime(),
                 IS_OVERSHADOWED_FALSE,
                 // there is an assumption here that unpublished segments are never overshadowed
                 segment.getShardSpec(),
@@ -450,8 +434,8 @@ public class SystemSchema extends AbstractSchema
       return authorizedSegments.iterator();
     }
 
-    private Iterator<Entry<SegmentId, AvailableSegmentMetadata>> getAuthorizedAvailableSegments(
-        Iterator<Entry<SegmentId, AvailableSegmentMetadata>> availableSegmentEntries,
+    private Iterator<AvailableSegmentMetadata> getAuthorizedAvailableSegments(
+        Iterator<AvailableSegmentMetadata> availableSegmentEntries,
         DataContext root
     )
     {
@@ -460,12 +444,12 @@ public class SystemSchema extends AbstractSchema
           "authenticationResult in dataContext"
       );
 
-      Function<Entry<SegmentId, AvailableSegmentMetadata>, Iterable<ResourceAction>> raGenerator = segment ->
+      Function<AvailableSegmentMetadata, Iterable<ResourceAction>> raGenerator = segment ->
           Collections.singletonList(
-              AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getKey().getDataSource())
+              AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getSegment().getDataSource())
           );
 
-      final Iterable<Entry<SegmentId, AvailableSegmentMetadata>> authorizedSegments =
+      final Iterable<AvailableSegmentMetadata> authorizedSegments =
           AuthorizationUtils.filterAuthorizedResources(
               authenticationResult,
               () -> availableSegmentEntries,

From e28424ea25871d59d3c8e96cc8a5ea5d666690b3 Mon Sep 17 00:00:00 2001
From: Vishesh Garg <gargvishesh@gmail.com>
Date: Wed, 4 Sep 2024 16:28:04 +0530
Subject: [PATCH 84/99] Enable rollup on multi-value dimensions for compaction
 with MSQ engine (#16937)

Currently compaction with MSQ engine doesn't work for rollup on multi-value dimensions (MVDs), the reason being the default behaviour of grouping on MVD dimensions to unnest the dimension values; for instance grouping on `[s1,s2]` with aggregate `a` will result in two rows: `<s1,a>` and `<s2,a>`.

This change enables rollup on MVDs (without unnest) by converting MVDs to Arrays before rollup using virtual columns, and then converting them back to MVDs using post aggregators. If segment schema is available to the compaction task (when it ends up downloading segments to get existing dimensions/metrics/granularity), it selectively does the MVD-Array conversion only for known multi-valued columns; else it conservatively performs this conversion for all `string` columns.
---
 .../msq/indexing/MSQCompactionRunner.java     | 186 +++++++++++++-----
 .../msq/indexing/MSQCompactionRunnerTest.java |  79 ++++++--
 .../indexing/common/task/CompactionTask.java  |  67 ++++++-
 .../common/task/CompactionTaskTest.java       |  70 +++++--
 .../duty/ITAutoCompactionTest.java            |  57 +++++-
 .../json/wikipedia_index_data1.json           |   6 +-
 .../json/wikipedia_index_data2.json           |   4 +-
 .../json/wikipedia_index_data3.json           |   6 +-
 .../indexer/wikipedia_index_task.json         |   1 +
 .../segment/indexing/CombinedDataSchema.java  |  67 +++++++
 .../segment/indexing/DataSchemaTest.java      |  22 +++
 11 files changed, 461 insertions(+), 104 deletions(-)
 create mode 100644 server/src/main/java/org/apache/druid/segment/indexing/CombinedDataSchema.java

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
index 5457e04286c..dc19e33282c 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
@@ -54,19 +54,21 @@ import org.apache.druid.query.QueryContext;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.query.TableDataSource;
 import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.query.aggregation.PostAggregator;
+import org.apache.druid.query.aggregation.post.ExpressionPostAggregator;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.DimensionSpec;
-import org.apache.druid.query.expression.TimestampFloorExprMacro;
-import org.apache.druid.query.expression.TimestampParseExprMacro;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
 import org.apache.druid.query.groupby.orderby.OrderByColumnSpec;
 import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
+import org.apache.druid.segment.VirtualColumn;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.indexing.CombinedDataSchema;
 import org.apache.druid.segment.indexing.DataSchema;
 import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
 import org.apache.druid.server.coordinator.CompactionConfigValidationResult;
@@ -82,6 +84,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -92,12 +95,14 @@ public class MSQCompactionRunner implements CompactionRunner
   private static final Granularity DEFAULT_SEGMENT_GRANULARITY = Granularities.ALL;
 
   private final ObjectMapper jsonMapper;
+  private final ExprMacroTable exprMacroTable;
   private final Injector injector;
   // Needed as output column name while grouping in the scenario of:
   // a) no query granularity -- to specify an output name for the time dimension column since __time is a reserved name.
   // b) custom query granularity -- to create a virtual column containing the rounded-off row timestamp.
   // In both cases, the new column is converted back to __time later using columnMappings.
   public static final String TIME_VIRTUAL_COLUMN = "__vTime";
+  public static final String ARRAY_VIRTUAL_COLUMN_PREFIX = "__vArray_";
 
   @JsonIgnore
   private final CurrentSubTaskHolder currentSubTaskHolder = new CurrentSubTaskHolder(
@@ -108,9 +113,14 @@ public class MSQCompactionRunner implements CompactionRunner
 
 
   @JsonCreator
-  public MSQCompactionRunner(@JacksonInject ObjectMapper jsonMapper, @JacksonInject Injector injector)
+  public MSQCompactionRunner(
+      @JacksonInject final ObjectMapper jsonMapper,
+      @JacksonInject final ExprMacroTable exprMacroTable,
+      @JacksonInject final Injector injector
+  )
   {
     this.jsonMapper = jsonMapper;
+    this.exprMacroTable = exprMacroTable;
     this.injector = injector;
   }
 
@@ -192,11 +202,12 @@ public class MSQCompactionRunner implements CompactionRunner
       Query<?> query;
       Interval interval = intervalDataSchema.getKey();
       DataSchema dataSchema = intervalDataSchema.getValue();
+      Map<String, VirtualColumn> inputColToVirtualCol = getVirtualColumns(dataSchema, interval);
 
       if (isGroupBy(dataSchema)) {
-        query = buildGroupByQuery(compactionTask, interval, dataSchema);
+        query = buildGroupByQuery(compactionTask, interval, dataSchema, inputColToVirtualCol);
       } else {
-        query = buildScanQuery(compactionTask, interval, dataSchema);
+        query = buildScanQuery(compactionTask, interval, dataSchema, inputColToVirtualCol);
       }
       QueryContext compactionTaskContext = new QueryContext(compactionTask.getContext());
 
@@ -308,7 +319,10 @@ public class MSQCompactionRunner implements CompactionRunner
     return rowSignatureBuilder.build();
   }
 
-  private static List<DimensionSpec> getAggregateDimensions(DataSchema dataSchema)
+  private static List<DimensionSpec> getAggregateDimensions(
+      DataSchema dataSchema,
+      Map<String, VirtualColumn> inputColToVirtualCol
+  )
   {
     List<DimensionSpec> dimensionSpecs = new ArrayList<>();
 
@@ -319,14 +333,22 @@ public class MSQCompactionRunner implements CompactionRunner
       // The changed granularity would result in a new virtual column that needs to be aggregated upon.
       dimensionSpecs.add(new DefaultDimensionSpec(TIME_VIRTUAL_COLUMN, TIME_VIRTUAL_COLUMN, ColumnType.LONG));
     }
-
-    dimensionSpecs.addAll(dataSchema.getDimensionsSpec().getDimensions().stream()
-                                    .map(dim -> new DefaultDimensionSpec(
-                                        dim.getName(),
-                                        dim.getName(),
-                                        dim.getColumnType()
-                                    ))
-                                    .collect(Collectors.toList()));
+    // If virtual columns are created from dimensions, replace dimension columns names with virtual column names.
+    dimensionSpecs.addAll(
+        dataSchema.getDimensionsSpec().getDimensions().stream()
+                  .map(dim -> {
+                    String dimension = dim.getName();
+                    ColumnType colType = dim.getColumnType();
+                    if (inputColToVirtualCol.containsKey(dim.getName())) {
+                      VirtualColumn virtualColumn = inputColToVirtualCol.get(dimension);
+                      dimension = virtualColumn.getOutputName();
+                      if (virtualColumn instanceof ExpressionVirtualColumn) {
+                        colType = ((ExpressionVirtualColumn) virtualColumn).getOutputType();
+                      }
+                    }
+                    return new DefaultDimensionSpec(dimension, dimension, colType);
+                  })
+                  .collect(Collectors.toList()));
     return dimensionSpecs;
   }
 
@@ -365,13 +387,19 @@ public class MSQCompactionRunner implements CompactionRunner
     return Collections.emptyList();
   }
 
-  private static Query<?> buildScanQuery(CompactionTask compactionTask, Interval interval, DataSchema dataSchema)
+  private static Query<?> buildScanQuery(
+      CompactionTask compactionTask,
+      Interval interval,
+      DataSchema dataSchema,
+      Map<String, VirtualColumn> inputColToVirtualCol
+  )
   {
     RowSignature rowSignature = getRowSignature(dataSchema);
+    VirtualColumns virtualColumns = VirtualColumns.create(new ArrayList<>(inputColToVirtualCol.values()));
     Druids.ScanQueryBuilder scanQueryBuilder = new Druids.ScanQueryBuilder()
         .dataSource(dataSchema.getDataSource())
         .columns(rowSignature.getColumnNames())
-        .virtualColumns(getVirtualColumns(dataSchema, interval))
+        .virtualColumns(virtualColumns)
         .columnTypes(rowSignature.getColumnTypes())
         .intervals(new MultipleIntervalSegmentSpec(Collections.singletonList(interval)))
         .filters(dataSchema.getTransformSpec().getFilter())
@@ -416,51 +444,115 @@ public class MSQCompactionRunner implements CompactionRunner
   }
 
   /**
-   * Creates a virtual timestamp column to create a new __time field according to the provided queryGranularity, as
-   * queryGranularity field itself is mandated to be ALL in MSQControllerTask.
+   * Conditionally creates below virtual columns
+   * <ul>
+   * <li>timestamp column (for custom queryGranularity): converts __time field in line with the provided
+   * queryGranularity, since the queryGranularity field itself in MSQControllerTask is mandated to be ALL.</li>
+   * <li>mv_to_array columns (for group-by queries): temporary columns that convert MVD columns to array to enable
+   * grouping on them without unnesting.</li>
+   * </ul>
    */
-  private static VirtualColumns getVirtualColumns(DataSchema dataSchema, Interval interval)
+  private Map<String, VirtualColumn> getVirtualColumns(DataSchema dataSchema, Interval interval)
   {
-    if (isQueryGranularityEmptyOrNone(dataSchema)) {
-      return VirtualColumns.EMPTY;
+    Map<String, VirtualColumn> inputColToVirtualCol = new HashMap<>();
+    if (!isQueryGranularityEmptyOrNone(dataSchema)) {
+      // Round-off time field according to provided queryGranularity
+      String timeVirtualColumnExpr;
+      if (dataSchema.getGranularitySpec()
+                    .getQueryGranularity()
+                    .equals(Granularities.ALL)) {
+        // For ALL query granularity, all records in a segment are assigned the interval start timestamp of the segment.
+        // It's the same behaviour in native compaction.
+        timeVirtualColumnExpr = StringUtils.format("timestamp_parse('%s')", interval.getStart());
+      } else {
+        PeriodGranularity periodQueryGranularity = (PeriodGranularity) dataSchema.getGranularitySpec()
+                                                                                 .getQueryGranularity();
+        // Round off the __time column according to the required granularity.
+        timeVirtualColumnExpr =
+            StringUtils.format(
+                "timestamp_floor(\"%s\", '%s')",
+                ColumnHolder.TIME_COLUMN_NAME,
+                periodQueryGranularity.getPeriod().toString()
+            );
+      }
+      inputColToVirtualCol.put(ColumnHolder.TIME_COLUMN_NAME, new ExpressionVirtualColumn(
+          TIME_VIRTUAL_COLUMN,
+          timeVirtualColumnExpr,
+          ColumnType.LONG,
+          exprMacroTable
+      ));
     }
-    String virtualColumnExpr;
-    if (dataSchema.getGranularitySpec()
-                  .getQueryGranularity()
-                  .equals(Granularities.ALL)) {
-      // For ALL query granularity, all records in a segment are assigned the interval start timestamp of the segment.
-      // It's the same behaviour in native compaction.
-      virtualColumnExpr = StringUtils.format("timestamp_parse('%s')", interval.getStart());
-    } else {
-      PeriodGranularity periodQueryGranularity = (PeriodGranularity) dataSchema.getGranularitySpec()
-                                                                               .getQueryGranularity();
-      // Round of the __time column according to the required granularity.
-      virtualColumnExpr =
-          StringUtils.format(
-              "timestamp_floor(\"%s\", '%s')",
-              ColumnHolder.TIME_COLUMN_NAME,
-              periodQueryGranularity.getPeriod().toString()
-          );
+    if (isGroupBy(dataSchema)) {
+      // Convert MVDs to arrays for grouping to avoid unnest, assuming all string cols to be MVDs.
+      Set<String> multiValuedColumns = dataSchema.getDimensionsSpec()
+                                                 .getDimensions()
+                                                 .stream()
+                                                 .filter(dim -> dim.getColumnType().equals(ColumnType.STRING))
+                                                 .map(DimensionSchema::getName)
+                                                 .collect(Collectors.toSet());
+      if (dataSchema instanceof CombinedDataSchema &&
+          ((CombinedDataSchema) dataSchema).getMultiValuedDimensions() != null) {
+        // Filter actual MVDs from schema info.
+        Set<String> multiValuedColumnsFromSchema =
+            ((CombinedDataSchema) dataSchema).getMultiValuedDimensions();
+        multiValuedColumns = multiValuedColumns.stream()
+                                               .filter(multiValuedColumnsFromSchema::contains)
+                                               .collect(Collectors.toSet());
+      }
+
+      for (String dim : multiValuedColumns) {
+        String virtualColumnExpr = StringUtils.format("mv_to_array(\"%s\")", dim);
+        inputColToVirtualCol.put(
+            dim,
+            new ExpressionVirtualColumn(
+                ARRAY_VIRTUAL_COLUMN_PREFIX + dim,
+                virtualColumnExpr,
+                ColumnType.STRING_ARRAY,
+                exprMacroTable
+            )
+        );
+      }
     }
-    return VirtualColumns.create(new ExpressionVirtualColumn(
-        TIME_VIRTUAL_COLUMN,
-        virtualColumnExpr,
-        ColumnType.LONG,
-        new ExprMacroTable(ImmutableList.of(new TimestampFloorExprMacro(), new TimestampParseExprMacro()))
-    ));
+    return inputColToVirtualCol;
   }
 
-  private static Query<?> buildGroupByQuery(CompactionTask compactionTask, Interval interval, DataSchema dataSchema)
+  private Query<?> buildGroupByQuery(
+      CompactionTask compactionTask,
+      Interval interval,
+      DataSchema dataSchema,
+      Map<String, VirtualColumn> inputColToVirtualCol
+  )
   {
     DimFilter dimFilter = dataSchema.getTransformSpec().getFilter();
 
+    VirtualColumns virtualColumns = VirtualColumns.create(new ArrayList<>(inputColToVirtualCol.values()));
+
+    // Convert MVDs converted to arrays back to MVDs, with the same name as the input column.
+    // This is safe since input column names no longer exist at post-aggregation stage.
+    List<PostAggregator> postAggregators =
+        inputColToVirtualCol.entrySet()
+                            .stream()
+                            .filter(entry -> !entry.getKey().equals(ColumnHolder.TIME_COLUMN_NAME))
+                            .map(
+                                entry ->
+                                    new ExpressionPostAggregator(
+                                        entry.getKey(),
+                                        StringUtils.format("array_to_mv(\"%s\")", entry.getValue().getOutputName()),
+                                        null,
+                                        ColumnType.STRING,
+                                        exprMacroTable
+                                    )
+                            )
+                            .collect(Collectors.toList());
+
     GroupByQuery.Builder builder = new GroupByQuery.Builder()
         .setDataSource(new TableDataSource(compactionTask.getDataSource()))
-        .setVirtualColumns(getVirtualColumns(dataSchema, interval))
+        .setVirtualColumns(virtualColumns)
         .setDimFilter(dimFilter)
         .setGranularity(new AllGranularity())
-        .setDimensions(getAggregateDimensions(dataSchema))
+        .setDimensions(getAggregateDimensions(dataSchema, inputColToVirtualCol))
         .setAggregatorSpecs(Arrays.asList(dataSchema.getAggregators()))
+        .setPostAggregatorSpecs(postAggregators)
         .setContext(compactionTask.getContext())
         .setInterval(interval);
 
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
index 09c5ae47718..34092d061b2 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
@@ -20,7 +20,6 @@
 package org.apache.druid.msq.indexing;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.InjectableValues;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
@@ -44,7 +43,6 @@ import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.GranularityType;
-import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.msq.indexing.destination.DataSourceMSQDestination;
 import org.apache.druid.msq.kernel.WorkerAssignmentStrategy;
 import org.apache.druid.msq.util.MultiStageQueryContext;
@@ -52,19 +50,27 @@ import org.apache.druid.query.OrderBy;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.CountAggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable;
+import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.query.dimension.DimensionSpec;
+import org.apache.druid.query.expression.TestExprMacroTable;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.filter.SelectorDimFilter;
+import org.apache.druid.query.groupby.GroupByQuery;
 import org.apache.druid.query.scan.ScanQuery;
 import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.data.CompressionFactory;
 import org.apache.druid.segment.data.CompressionStrategy;
 import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
+import org.apache.druid.segment.indexing.CombinedDataSchema;
 import org.apache.druid.segment.indexing.DataSchema;
 import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
 import org.apache.druid.segment.transform.TransformSpec;
 import org.apache.druid.server.coordinator.CompactionConfigValidationResult;
 import org.apache.druid.sql.calcite.parser.DruidSqlInsert;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
 import org.joda.time.Interval;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -73,8 +79,10 @@ import org.junit.Test;
 import javax.annotation.Nullable;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -83,20 +91,25 @@ public class MSQCompactionRunnerTest
   private static final String DATA_SOURCE = "dataSource";
   private static final Interval COMPACTION_INTERVAL = Intervals.of("2017-01-01/2017-07-01");
 
-  private static final String TIMESTAMP_COLUMN = "timestamp";
+  private static final String TIMESTAMP_COLUMN = ColumnHolder.TIME_COLUMN_NAME;
   private static final int TARGET_ROWS_PER_SEGMENT = 100000;
   private static final GranularityType SEGMENT_GRANULARITY = GranularityType.HOUR;
   private static final GranularityType QUERY_GRANULARITY = GranularityType.HOUR;
   private static List<String> PARTITION_DIMENSIONS;
 
-  private static final StringDimensionSchema STRING_DIMENSION = new StringDimensionSchema("string_dim", null, null);
+  private static final StringDimensionSchema STRING_DIMENSION = new StringDimensionSchema("string_dim", null, false);
+  private static final StringDimensionSchema MV_STRING_DIMENSION = new StringDimensionSchema("mv_string_dim", null, null);
   private static final LongDimensionSchema LONG_DIMENSION = new LongDimensionSchema("long_dim");
-  private static final List<DimensionSchema> DIMENSIONS = ImmutableList.of(STRING_DIMENSION, LONG_DIMENSION);
+  private static final List<DimensionSchema> DIMENSIONS = ImmutableList.of(
+      STRING_DIMENSION,
+      LONG_DIMENSION,
+      MV_STRING_DIMENSION
+  );
   private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
   private static final AggregatorFactory AGG1 = new CountAggregatorFactory("agg_0");
   private static final AggregatorFactory AGG2 = new LongSumAggregatorFactory("sum_added", "sum_added");
   private static final List<AggregatorFactory> AGGREGATORS = ImmutableList.of(AGG1, AGG2);
-  private static final MSQCompactionRunner MSQ_COMPACTION_RUNNER = new MSQCompactionRunner(JSON_MAPPER, null);
+  private static final MSQCompactionRunner MSQ_COMPACTION_RUNNER = new MSQCompactionRunner(JSON_MAPPER, TestExprMacroTable.INSTANCE, null);
 
   @BeforeClass
   public static void setupClass()
@@ -110,11 +123,6 @@ public class MSQCompactionRunnerTest
     );
 
     PARTITION_DIMENSIONS = Collections.singletonList(stringDimensionSchema.getName());
-
-    JSON_MAPPER.setInjectableValues(new InjectableValues.Std().addValue(
-        ExprMacroTable.class,
-        LookupEnabledTestExprMacroTable.INSTANCE
-    ));
   }
 
   @Test
@@ -296,7 +304,10 @@ public class MSQCompactionRunnerTest
         actualMSQSpec.getDestination()
     );
 
-    Assert.assertEquals(dimFilter, actualMSQSpec.getQuery().getFilter());
+    Assert.assertTrue(actualMSQSpec.getQuery() instanceof ScanQuery);
+    ScanQuery scanQuery = (ScanQuery) actualMSQSpec.getQuery();
+
+    Assert.assertEquals(dimFilter, scanQuery.getFilter());
     Assert.assertEquals(
         JSON_MAPPER.writeValueAsString(SEGMENT_GRANULARITY.toString()),
         msqControllerTask.getContext().get(DruidSqlInsert.SQL_INSERT_SEGMENT_GRANULARITY)
@@ -305,7 +316,7 @@ public class MSQCompactionRunnerTest
     Assert.assertEquals(WorkerAssignmentStrategy.MAX, actualMSQSpec.getAssignmentStrategy());
     Assert.assertEquals(
         PARTITION_DIMENSIONS.stream().map(OrderBy::ascending).collect(Collectors.toList()),
-        ((ScanQuery) actualMSQSpec.getQuery()).getOrderBys()
+        scanQuery.getOrderBys()
     );
   }
 
@@ -322,7 +333,10 @@ public class MSQCompactionRunnerTest
         null
     );
 
-    DataSchema dataSchema = new DataSchema(
+    Set<String> multiValuedDimensions = new HashSet<>();
+    multiValuedDimensions.add(MV_STRING_DIMENSION.getName());
+
+    CombinedDataSchema dataSchema = new CombinedDataSchema(
         DATA_SOURCE,
         new TimestampSpec(TIMESTAMP_COLUMN, null, null),
         new DimensionsSpec(DIMENSIONS),
@@ -332,7 +346,8 @@ public class MSQCompactionRunnerTest
             QUERY_GRANULARITY.getDefaultGranularity(),
             Collections.singletonList(COMPACTION_INTERVAL)
         ),
-        new TransformSpec(dimFilter, Collections.emptyList())
+        new TransformSpec(dimFilter, Collections.emptyList()),
+        multiValuedDimensions
     );
 
 
@@ -367,7 +382,10 @@ public class MSQCompactionRunnerTest
         actualMSQSpec.getDestination()
     );
 
-    Assert.assertEquals(dimFilter, actualMSQSpec.getQuery().getFilter());
+    Assert.assertTrue(actualMSQSpec.getQuery() instanceof GroupByQuery);
+    GroupByQuery groupByQuery = (GroupByQuery) actualMSQSpec.getQuery();
+
+    Assert.assertEquals(dimFilter, groupByQuery.getFilter());
     Assert.assertEquals(
         JSON_MAPPER.writeValueAsString(SEGMENT_GRANULARITY.toString()),
         msqControllerTask.getContext().get(DruidSqlInsert.SQL_INSERT_SEGMENT_GRANULARITY)
@@ -377,6 +395,31 @@ public class MSQCompactionRunnerTest
         msqControllerTask.getContext().get(DruidSqlInsert.SQL_INSERT_QUERY_GRANULARITY)
     );
     Assert.assertEquals(WorkerAssignmentStrategy.MAX, actualMSQSpec.getAssignmentStrategy());
+
+
+    // Since only MV_STRING_DIMENSION is indicated to be MVD by the CombinedSchema, conversion to array should happen
+    // only for that column.
+    List<DimensionSpec> expectedDimensionSpec = DIMENSIONS.stream()
+                                                          .filter(dim -> !MV_STRING_DIMENSION.getName()
+                                                                                            .equals(dim.getName()))
+                                                          .map(dim -> new DefaultDimensionSpec(
+                                                              dim.getName(),
+                                                              dim.getName(),
+                                                              dim.getColumnType()
+                                                          ))
+                                                          .collect(
+                                                              Collectors.toList());
+    expectedDimensionSpec.add(
+        new DefaultDimensionSpec(MSQCompactionRunner.TIME_VIRTUAL_COLUMN,
+                                                       MSQCompactionRunner.TIME_VIRTUAL_COLUMN,
+                                                       ColumnType.LONG)
+    );
+    String mvToArrayStringDim = MSQCompactionRunner.ARRAY_VIRTUAL_COLUMN_PREFIX + MV_STRING_DIMENSION.getName();
+    expectedDimensionSpec.add(new DefaultDimensionSpec(mvToArrayStringDim, mvToArrayStringDim, ColumnType.STRING_ARRAY));
+    MatcherAssert.assertThat(
+        expectedDimensionSpec,
+        Matchers.containsInAnyOrder(groupByQuery.getDimensions().toArray(new DimensionSpec[0]))
+    );
   }
 
   private CompactionTask createCompactionTask(
@@ -408,7 +451,7 @@ public class MSQCompactionRunnerTest
         .transformSpec(transformSpec)
         .granularitySpec(granularitySpec)
         .metricsSpec(metricsSpec)
-        .compactionRunner(new MSQCompactionRunner(JSON_MAPPER, null))
+        .compactionRunner(MSQ_COMPACTION_RUNNER)
         .context(context);
 
     return builder.build();
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
index ab7736b047e..73c8a35405c 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
@@ -77,8 +77,10 @@ import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.IndexSpec;
 import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.incremental.AppendableIndexSpec;
+import org.apache.druid.segment.indexing.CombinedDataSchema;
 import org.apache.druid.segment.indexing.DataSchema;
 import org.apache.druid.segment.indexing.TuningConfig;
 import org.apache.druid.segment.indexing.granularity.GranularitySpec;
@@ -462,7 +464,8 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
         transformSpec,
         metricsSpec,
         granularitySpec,
-        getMetricBuilder()
+        getMetricBuilder(),
+        !(compactionRunner instanceof NativeCompactionRunner)
     );
 
     registerResourceCloserOnAbnormalExit(compactionRunner.getCurrentSubTaskHolder());
@@ -488,7 +491,8 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
       @Nullable final ClientCompactionTaskTransformSpec transformSpec,
       @Nullable final AggregatorFactory[] metricsSpec,
       @Nullable final ClientCompactionTaskGranularitySpec granularitySpec,
-      final ServiceMetricEvent.Builder metricBuilder
+      final ServiceMetricEvent.Builder metricBuilder,
+      boolean needMultiValuedColumns
   ) throws IOException
   {
     final Iterable<DataSegment> timelineSegments = retrieveRelevantTimelineHolders(
@@ -552,7 +556,8 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
             metricsSpec,
             granularitySpec == null
             ? new ClientCompactionTaskGranularitySpec(segmentGranularityToUse, null, null)
-            : granularitySpec.withSegmentGranularity(segmentGranularityToUse)
+            : granularitySpec.withSegmentGranularity(segmentGranularityToUse),
+            needMultiValuedColumns
         );
         intervalDataSchemaMap.put(interval, dataSchema);
       }
@@ -577,7 +582,8 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
           dimensionsSpec,
           transformSpec,
           metricsSpec,
-          granularitySpec
+          granularitySpec,
+          needMultiValuedColumns
       );
       return Collections.singletonMap(segmentProvider.interval, dataSchema);
     }
@@ -607,7 +613,8 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
       @Nullable DimensionsSpec dimensionsSpec,
       @Nullable ClientCompactionTaskTransformSpec transformSpec,
       @Nullable AggregatorFactory[] metricsSpec,
-      @Nonnull ClientCompactionTaskGranularitySpec granularitySpec
+      @Nonnull ClientCompactionTaskGranularitySpec granularitySpec,
+      boolean needMultiValuedColumns
   )
   {
     // Check index metadata & decide which values to propagate (i.e. carry over) for rollup & queryGranularity
@@ -616,7 +623,8 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
         granularitySpec.isRollup() == null,
         granularitySpec.getQueryGranularity() == null,
         dimensionsSpec == null,
-        metricsSpec == null
+        metricsSpec == null,
+        needMultiValuedColumns
     );
 
     final Stopwatch stopwatch = Stopwatch.createStarted();
@@ -668,13 +676,14 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
       finalMetricsSpec = metricsSpec;
     }
 
-    return new DataSchema(
+    return new CombinedDataSchema(
         dataSource,
         new TimestampSpec(ColumnHolder.TIME_COLUMN_NAME, "millis", null),
         finalDimensionsSpec,
         finalMetricsSpec,
         uniformGranularitySpec,
-        transformSpec == null ? null : new TransformSpec(transformSpec.getFilter(), null)
+        transformSpec == null ? null : new TransformSpec(transformSpec.getFilter(), null),
+        existingSegmentAnalyzer.getMultiValuedDimensions()
     );
   }
 
@@ -748,6 +757,7 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
     private final boolean needQueryGranularity;
     private final boolean needDimensionsSpec;
     private final boolean needMetricsSpec;
+    private final boolean needMultiValuedDimensions;
 
     // For processRollup:
     private boolean rollup = true;
@@ -761,13 +771,15 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
 
     // For processMetricsSpec:
     private final Set<List<AggregatorFactory>> aggregatorFactoryLists = new HashSet<>();
+    private Set<String> multiValuedDimensions;
 
     ExistingSegmentAnalyzer(
         final Iterable<Pair<DataSegment, Supplier<ResourceHolder<QueryableIndex>>>> segmentsIterable,
         final boolean needRollup,
         final boolean needQueryGranularity,
         final boolean needDimensionsSpec,
-        final boolean needMetricsSpec
+        final boolean needMetricsSpec,
+        final boolean needMultiValuedDimensions
     )
     {
       this.segmentsIterable = segmentsIterable;
@@ -775,15 +787,26 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
       this.needQueryGranularity = needQueryGranularity;
       this.needDimensionsSpec = needDimensionsSpec;
       this.needMetricsSpec = needMetricsSpec;
+      this.needMultiValuedDimensions = needMultiValuedDimensions;
+    }
+
+    private boolean shouldFetchSegments()
+    {
+      // Don't fetch segments for just needMultiValueDimensions
+      return needRollup || needQueryGranularity || needDimensionsSpec || needMetricsSpec;
     }
 
     public void fetchAndProcessIfNeeded()
     {
-      if (!needRollup && !needQueryGranularity && !needDimensionsSpec && !needMetricsSpec) {
+      if (!shouldFetchSegments()) {
         // Nothing to do; short-circuit and don't fetch segments.
         return;
       }
 
+      if (needMultiValuedDimensions) {
+        multiValuedDimensions = new HashSet<>();
+      }
+
       final List<Pair<DataSegment, Supplier<ResourceHolder<QueryableIndex>>>> segments = sortSegmentsListNewestFirst();
 
       for (Pair<DataSegment, Supplier<ResourceHolder<QueryableIndex>>> segmentPair : segments) {
@@ -804,6 +827,7 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
             processQueryGranularity(index);
             processDimensionsSpec(index);
             processMetricsSpec(index);
+            processMultiValuedDimensions(index);
           }
         }
       }
@@ -890,6 +914,11 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
       return mergedAggregators;
     }
 
+    public Set<String> getMultiValuedDimensions()
+    {
+      return multiValuedDimensions;
+    }
+
     /**
      * Sort {@link #segmentsIterable} in order, such that we look at later segments prior to earlier ones. Useful when
      * analyzing dimensions, as it allows us to take the latest value we see, and therefore prefer types from more
@@ -983,6 +1012,24 @@ public class CompactionTask extends AbstractBatchIndexTask implements PendingSeg
       }
     }
 
+    private void processMultiValuedDimensions(final QueryableIndex index)
+    {
+      if (!needMultiValuedDimensions) {
+        return;
+      }
+      for (String dimension : index.getAvailableDimensions()) {
+        if (isMultiValuedDimension(index, dimension)) {
+          multiValuedDimensions.add(dimension);
+        }
+      }
+    }
+
+    private boolean isMultiValuedDimension(final QueryableIndex index, final String col)
+    {
+      ColumnCapabilities columnCapabilities = index.getColumnCapabilities(col);
+      return columnCapabilities != null && columnCapabilities.hasMultipleValues().isTrue();
+    }
+
     static Granularity compareWithCurrent(Granularity queryGranularity, Granularity current)
     {
       if (queryGranularity == null && current != null) {
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
index 84a1457ad91..c1bf649980f 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
@@ -116,6 +116,7 @@ import org.apache.druid.segment.data.ListIndexed;
 import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
 import org.apache.druid.segment.incremental.RowIngestionMetersFactory;
 import org.apache.druid.segment.indexing.BatchIOConfig;
+import org.apache.druid.segment.indexing.CombinedDataSchema;
 import org.apache.druid.segment.indexing.DataSchema;
 import org.apache.druid.segment.indexing.TuningConfig;
 import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
@@ -749,7 +750,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -810,7 +812,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -872,7 +875,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -935,7 +939,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1005,7 +1010,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1055,7 +1061,8 @@ public class CompactionTaskTest
         null,
         customMetricsSpec,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1098,7 +1105,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1148,7 +1156,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     NativeCompactionRunner.createIngestionSpecs(
@@ -1178,7 +1187,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     NativeCompactionRunner.createIngestionSpecs(
@@ -1219,7 +1229,8 @@ public class CompactionTaskTest
         null,
         null,
         new ClientCompactionTaskGranularitySpec(new PeriodGranularity(Period.months(3), null, null), null, null),
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1263,7 +1274,8 @@ public class CompactionTaskTest
         null,
         null,
         new ClientCompactionTaskGranularitySpec(null, new PeriodGranularity(Period.months(3), null, null), null),
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
         dataSchemasForIntervals,
@@ -1308,7 +1320,8 @@ public class CompactionTaskTest
             new PeriodGranularity(Period.months(3), null, null),
             null
         ),
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1355,7 +1368,8 @@ public class CompactionTaskTest
         null,
         null,
         null,
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1400,7 +1414,8 @@ public class CompactionTaskTest
         null,
         null,
         new ClientCompactionTaskGranularitySpec(null, null, null),
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1445,7 +1460,8 @@ public class CompactionTaskTest
         null,
         null,
         new ClientCompactionTaskGranularitySpec(null, null, true),
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1475,7 +1491,8 @@ public class CompactionTaskTest
         null,
         null,
         new ClientCompactionTaskGranularitySpec(null, null, null),
-        METRIC_BUILDER
+        METRIC_BUILDER,
+        false
     );
 
     final List<ParallelIndexIngestionSpec> ingestionSpecs = NativeCompactionRunner.createIngestionSpecs(
@@ -1495,6 +1512,27 @@ public class CompactionTaskTest
     }
   }
 
+  @Test
+  public void testMultiValuedDimensionsProcessing()
+      throws IOException
+  {
+    final Map<Interval, DataSchema> dataSchemasForIntervals = CompactionTask.createDataSchemasForIntervals(
+        toolbox,
+        LockGranularity.TIME_CHUNK,
+        new SegmentProvider(DATA_SOURCE, new CompactionIntervalSpec(COMPACTION_INTERVAL, null)),
+        null,
+        null,
+        null,
+        new ClientCompactionTaskGranularitySpec(null, null, null),
+        METRIC_BUILDER,
+        true
+    );
+    for (DataSchema dataSchema : dataSchemasForIntervals.values()) {
+      Assert.assertTrue(dataSchema instanceof CombinedDataSchema);
+      Assert.assertTrue(((CombinedDataSchema) dataSchema).getMultiValuedDimensions().isEmpty());
+    }
+  }
+
   @Test
   public void testChooseFinestGranularityWithNulls()
   {
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
index 31a6bccffc7..e95d09bd508 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
@@ -526,7 +526,7 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
     try (final Closeable ignored = unloader(fullDatasourceName)) {
       final List<String> intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName);
       intervalsBeforeCompaction.sort(null);
-      // 4 segments across 2 days (4 total)...
+      // 4 segments across 2 days (4 total)
       verifySegmentsCount(4);
       verifyQuery(INDEX_QUERIES_RESOURCE);
 
@@ -548,13 +548,59 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
           true,
           engine
       );
-      //...compacted into 1 segment for the entire year.
+      // Compacted into 1 segment for the entire year.
       forceTriggerAutoCompaction(1);
       verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED);
       verifySegmentsCompactedDimensionSchema(dimensionSchemas);
     }
   }
 
+  @Test(dataProvider = "engine")
+  public void testAutoCompactionRollsUpMultiValueDimensionsWithoutUnnest(CompactionEngine engine) throws Exception
+  {
+    loadData(INDEX_TASK);
+    try (final Closeable ignored = unloader(fullDatasourceName)) {
+      final List<String> intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName);
+      intervalsBeforeCompaction.sort(null);
+      // 4 segments across 2 days (4 total)
+      verifySegmentsCount(4);
+      verifyQuery(INDEX_QUERIES_RESOURCE);
+
+      LOG.info("Auto compaction test with YEAR segment granularity, DAY query granularity, dropExisting is true");
+
+      List<DimensionSchema> dimensionSchemas = ImmutableList.of(
+          new StringDimensionSchema("language", null, true),
+          new StringDimensionSchema("tags", DimensionSchema.MultiValueHandling.SORTED_ARRAY, true)
+      );
+
+      submitCompactionConfig(
+          MAX_ROWS_PER_SEGMENT_COMPACTED,
+          NO_SKIP_OFFSET,
+          new UserCompactionTaskGranularityConfig(Granularities.YEAR, Granularities.DAY, true),
+          new UserCompactionTaskDimensionsConfig(dimensionSchemas),
+          null,
+          new AggregatorFactory[] {new LongSumAggregatorFactory("added", "added")},
+          true,
+          engine
+      );
+      // Compacted into 1 segment for the entire year.
+      forceTriggerAutoCompaction(1);
+      Map<String, Object> queryAndResultFields = ImmutableMap.of(
+          "%%FIELD_TO_QUERY%%",
+          "added",
+          "%%EXPECTED_COUNT_RESULT%%",
+          1,
+          "%%EXPECTED_SCAN_RESULT%%",
+          ImmutableList.of(
+              ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516)))
+          )
+      );
+      verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, queryAndResultFields);
+      verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED);
+      verifySegmentsCompactedDimensionSchema(dimensionSchemas);
+    }
+  }
+
   @Test
   public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception
   {
@@ -636,10 +682,11 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
 
         final HashedPartitionsSpec hashedPartitionsSpec = new HashedPartitionsSpec(null, 3, null);
         submitCompactionConfig(hashedPartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, null, false, engine);
-        // 2 segments published per day after compaction.
-        forceTriggerAutoCompaction(4);
+        // 3 segments for both 2013-08-31 and 2013-09-01. (Note that numShards guarantees max shards but not exact
+        // number of final shards, since some shards may end up empty.)
+        forceTriggerAutoCompaction(6);
         verifyQuery(INDEX_QUERIES_RESOURCE);
-        verifySegmentsCompacted(hashedPartitionsSpec, 4);
+        verifySegmentsCompacted(hashedPartitionsSpec, 6);
         checkCompactionIntervals(intervalsBeforeCompaction);
       }
 
diff --git a/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data1.json b/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data1.json
index 169796cd746..30fef106aaf 100644
--- a/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data1.json
+++ b/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data1.json
@@ -1,3 +1,3 @@
-{"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143}
-{"timestamp": "2013-08-31T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330}
-{"timestamp": "2013-08-31T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111}
\ No newline at end of file
+{"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "tags": ["t1", "t2"], "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143}
+{"timestamp": "2013-08-31T03:32:45Z", "page": "Striker Eureka", "language" : "en", "tags": ["t1", "t2"], "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330}
+{"timestamp": "2013-08-31T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "tags": ["t3", "t4", "t5"], "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111}
\ No newline at end of file
diff --git a/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data2.json b/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data2.json
index 62e270113d5..d13b6a37bbc 100644
--- a/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data2.json
+++ b/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data2.json
@@ -1,3 +1,3 @@
-{"timestamp": "2013-08-31T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900}
+{"timestamp": "2013-08-31T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "tags": ["t6"], "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900}
 {"timestamp": "2013-08-31T12:41:27Z", "page": "Coyote Tango", "language" : "ja", "user" : "stringer", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo", "added": 1, "deleted": 10, "delta": -9}
-{"timestamp": "2013-09-01T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143}
\ No newline at end of file
+{"timestamp": "2013-09-01T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "tags": ["t1", "t2"], "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143}
\ No newline at end of file
diff --git a/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data3.json b/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data3.json
index 28e0762f84c..a8f5c2ec292 100644
--- a/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data3.json
+++ b/integration-tests/src/test/resources/data/batch_index/json/wikipedia_index_data3.json
@@ -1,4 +1,4 @@
-{"timestamp": "2013-09-01T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330}
-{"timestamp": "2013-09-01T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111}
-{"timestamp": "2013-09-01T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900}
+{"timestamp": "2013-09-01T03:32:45Z", "page": "Striker Eureka", "language" : "en", "tags": ["t1", "t2"], "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330}
+{"timestamp": "2013-09-01T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "tags": ["t3", "t4", "t5"], "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111}
+{"timestamp": "2013-09-01T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "tags": ["t6"], "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900}
 {"timestamp": "2013-09-01T12:41:27Z", "page": "Coyote Tango", "language" : "ja", "user" : "stringer", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo", "added": 1, "deleted": 10, "delta": -9}
\ No newline at end of file
diff --git a/integration-tests/src/test/resources/indexer/wikipedia_index_task.json b/integration-tests/src/test/resources/indexer/wikipedia_index_task.json
index 00bf7721f2b..00005c7b65a 100644
--- a/integration-tests/src/test/resources/indexer/wikipedia_index_task.json
+++ b/integration-tests/src/test/resources/indexer/wikipedia_index_task.json
@@ -11,6 +11,7 @@
                 "dimensions": [
                     "page",
                     {"type": "string", "name": "language", "createBitmapIndex": false},
+                    "tags",
                     "user",
                     "unpatrolled",
                     "newPage",
diff --git a/server/src/main/java/org/apache/druid/segment/indexing/CombinedDataSchema.java b/server/src/main/java/org/apache/druid/segment/indexing/CombinedDataSchema.java
new file mode 100644
index 00000000000..14deba7725a
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/segment/indexing/CombinedDataSchema.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.indexing;
+
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.segment.indexing.granularity.GranularitySpec;
+import org.apache.druid.segment.transform.TransformSpec;
+
+import javax.annotation.Nullable;
+import java.util.Set;
+
+/**
+ * Class representing the combined DataSchema of a set of segments, currently used only by Compaction.
+ */
+public class CombinedDataSchema extends DataSchema
+{
+  private final Set<String> multiValuedDimensions;
+
+  public CombinedDataSchema(
+      String dataSource,
+      @Nullable TimestampSpec timestampSpec,
+      @Nullable DimensionsSpec dimensionsSpec,
+      AggregatorFactory[] aggregators,
+      GranularitySpec granularitySpec,
+      TransformSpec transformSpec,
+      @Nullable Set<String> multiValuedDimensions
+  )
+  {
+    super(
+        dataSource,
+        timestampSpec,
+        dimensionsSpec,
+        aggregators,
+        granularitySpec,
+        transformSpec,
+        null,
+        null
+    );
+    this.multiValuedDimensions = multiValuedDimensions;
+  }
+
+  @Nullable
+  public Set<String> getMultiValuedDimensions()
+  {
+    return multiValuedDimensions;
+  }
+
+}
diff --git a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
index 32b2a3830fd..90297dd4af9 100644
--- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
+++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
@@ -779,4 +779,26 @@ public class DataSchemaTest extends InitializedNullHandlingTest
     Assert.assertSame(oldSchema.getParserMap(), newSchema.getParserMap());
 
   }
+
+  @Test
+  public void testCombinedDataSchemaSetsMultiValuedColumnsInfo()
+  {
+    Set<String> multiValuedDimensions = ImmutableSet.of("dimA");
+
+    CombinedDataSchema schema = new CombinedDataSchema(
+        IdUtilsTest.VALID_ID_CHARS,
+        new TimestampSpec("time", "auto", null),
+        DimensionsSpec.builder()
+                      .setDimensions(
+                          DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimA", "dimB", "metric1"))
+                      )
+                      .setDimensionExclusions(ImmutableList.of("dimC"))
+                      .build(),
+        null,
+        new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))),
+        null,
+        multiValuedDimensions
+    );
+    Assert.assertEquals(ImmutableSet.of("dimA"), schema.getMultiValuedDimensions());
+  }
 }

From b698440bfe667c74bffb277350bafb0d126baed6 Mon Sep 17 00:00:00 2001
From: Laksh Singla <lakshsingla@gmail.com>
Date: Wed, 4 Sep 2024 19:37:23 +0530
Subject: [PATCH 85/99] suppress cve (#16997)

---
 owasp-dependency-check-suppressions.xml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/owasp-dependency-check-suppressions.xml b/owasp-dependency-check-suppressions.xml
index d25148eb0de..283e882183f 100644
--- a/owasp-dependency-check-suppressions.xml
+++ b/owasp-dependency-check-suppressions.xml
@@ -699,4 +699,13 @@
     ]]></notes>
     <vulnerabilityName>CVE-2024-25638</vulnerabilityName>
   </suppress>
+  <suppress>
+    <!-- The CVE is also not applicable to xz-java because it does not implement xzgrep and therefore is not vulnerable
+     ~ to the filename validation problem. Druid does not use xzgrep but this CVE is popping up because the CPE matches the
+     ~ Java package too. -->
+    <notes><![CDATA[
+    file name: xz-1.9.jar
+    ]]></notes>
+    <vulnerabilityName>CVE-2022-1271</vulnerabilityName>
+  </suppress>
 </suppressions>

From 03c37b3143451635b60f8ac39edff4665507135a Mon Sep 17 00:00:00 2001
From: Katya Macedo <38017980+ektravel@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:33:17 -0500
Subject: [PATCH 86/99] Fix spelling (#17001)

---
 docs/assets/druid-architecture.svg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/assets/druid-architecture.svg b/docs/assets/druid-architecture.svg
index 3f86a412cfa..8db6ec25d4a 100644
--- a/docs/assets/druid-architecture.svg
+++ b/docs/assets/druid-architecture.svg
@@ -16,4 +16,4 @@
   ~ specific language governing permissions and limitations
   ~ under the License.
   -->
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:lucid="lucid" width="1967.24" height="864.83"><g transform="translate(5341 -19.5)" lucid:page-tab-id="0_0"><path d="M-5320 166a6 6 0 0 1 6-6h508a6 6 0 0 1 6 6v308a6 6 0 0 1-6 6h-508a6 6 0 0 1-6-6zM-4640 166a6 6 0 0 1 6-6h508a6 6 0 0 1 6 6v308a6 6 0 0 1-6 6h-508a6 6 0 0 1-6-6zM-3960 166a6 6 0 0 1 6-6h508a6 6 0 0 1 6 6v308a6 6 0 0 1-6 6h-508a6 6 0 0 1-6-6z" stroke="#cfe4ff" stroke-width="2" fill="#cfe4ff"/><path d="M-5280 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#a" transform="matrix(1,0,0,1,-5275,265) translate(9.728395061728392 60.923611111111114)"/><path d="M-5020 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#b" transform="matrix(1,0,0,1,-5015,265) translate(29.750000000000007 60.923611111111114)"/><path d="M-3920 266a6 6 0 0 1 6-6h208a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-208a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#c" transform="matrix(1,0,0,1,-3915,265) translate(5.694444444444429 60.923611111111114)"/><path d="M-4340 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#d" transform="matrix(1,0,0,1,-4335,265) translate(42.54938271604939 60.923611111111114)"/><path d="M-4600 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#e" transform="matrix(1,0,0,1,-4595,265) translate(41.74691358024692 60.923611111111114)"/><path d="M-3660 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#f" transform="matrix(1,0,0,1,-3655,265) translate(25.817901234567906 60.923611111111114)"/><path d="M-4401 320h42" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4416.26 320l14.26-4.63v9.26zM-4343.74 320l-14.26 4.63v-9.26z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-5320 156.33a6 6 0 0 1 6-6h208a6 6 0 0 1 6 6v67.34a6 6 0 0 1-6 6h-208a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#g" transform="matrix(1,0,0,1,-5315,155.33333333333334) translate(11.191358024691354 40.611111111111114)"/><use xlink:href="#h" transform="matrix(1,0,0,1,-5315,155.33333333333334) translate(112.14197530864197 40.611111111111114)"/><path d="M-4640 166a6 6 0 0 1 6-6h188a6 6 0 0 1 6 6v48a6 6 0 0 1-6 6h-188a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#i" transform="matrix(1,0,0,1,-4635,165.00000000000006) translate(5.966049382716051 29.36111111111111)"/><use xlink:href="#h" transform="matrix(1,0,0,1,-4635,165.00000000000006) translate(97.36728395061729 29.36111111111111)"/><path d="M-3960 166a6 6 0 0 1 6-6h188a6 6 0 0 1 6 6v48a6 6 0 0 1-6 6h-188a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#j" transform="matrix(1,0,0,1,-3955,165.00000000000006) translate(16.43827160493828 29.36111111111111)"/><use xlink:href="#h" transform="matrix(1,0,0,1,-3955,165.00000000000006) translate(86.89506172839506 29.36111111111111)"/><path d="M-5000 646a6 6 0 0 1 6-6h1228a6 6 0 0 1 6 6v188a6 6 0 0 1-6 6h-1228a6 6 0 0 1-6-6z" stroke="#ced4db" fill="#f2f3f5"/><use xlink:href="#k" transform="matrix(1,0,0,1,-4988,652) translate(567.9074074074074 94.77777777777777)"/><path d="M-4530 686a6 6 0 0 1 6-6h288a6 6 0 0 1 6 6v68a6 6 0 0 1-6 6h-288a6 6 0 0 1-6-6z" stroke="#9391ff" fill="#dedeff"/><use xlink:href="#l" transform="matrix(1,0,0,1,-4518,692) translate(66.6604938271605 33.611111111111114)"/><path d="M-4120 686a6 6 0 0 1 6-6h288a6 6 0 0 1 6 6v68a6 6 0 0 1-6 6h-288a6 6 0 0 1-6-6z" stroke="#9391ff" fill="#dedeff"/><use xlink:href="#m" transform="matrix(1,0,0,1,-4108,692) translate(51.4135802469136 33.611111111111114)"/><use xlink:href="#n" transform="matrix(1,0,0,1,-4108,692) translate(128.37037037037038 33.611111111111114)"/><path d="M-4940 686a6 6 0 0 1 6-6h288a6 6 0 0 1 6 6v68a6 6 0 0 1-6 6h-288a6 6 0 0 1-6-6z" stroke="#9391ff" fill="#dedeff"/><use xlink:href="#o" transform="matrix(1,0,0,1,-4928,692) translate(25.734567901234584 33.611111111111114)"/><use xlink:href="#n" transform="matrix(1,0,0,1,-4928,692) translate(154.0493827160494 33.611111111111114)"/><path d="M-4560 766.67a6 6 0 0 1 6-6h348a6 6 0 0 1 6 6V834a6 6 0 0 1-6 6h-348a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#p" transform="matrix(1,0,0,1,-4555,765.6666666666666) translate(17.99691358024691 40.611111111111114)"/><use xlink:href="#q" transform="matrix(1,0,0,1,-4555,765.6666666666666) translate(139.81172839506172 40.611111111111114)"/><path d="M-4387.38 662.53l-1.6-3.68m-3.2-7.36l-3.2-7.38m-3.2-7.36l-3.2-7.36m-3.2-7.36l-3.2-7.37m-3.2-7.36l-3.2-7.35m-3.2-7.37l-3.2-7.36m-3.2-7.36l-3.2-7.36m-3.2-7.37l-3.22-7.36m-3.2-7.36l-3.2-7.36m-3.2-7.36l-3.2-7.36m-3.2-7.36l-3.2-7.37m-3.2-7.37l-3.2-7.36m-3.2-7.37l-3.2-7.36m-3.2-7.36l-3.2-7.36m-3.2-7.37l-3.2-7.36m-3.2-7.37l-3.22-7.37m-3.2-7.36l-3.2-7.36m-3.2-7.37l-3.2-7.36m-3.2-7.35l-1.6-3.68" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4381.3 676.53l-9.93-11.23 8.5-3.7zM-4508.7 383.47l9.93 11.23-8.5 3.7z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4372.62 662.53l1.6-3.68m3.2-7.36l3.2-7.38m3.2-7.36l3.2-7.36m3.2-7.36l3.2-7.37m3.2-7.36l3.2-7.35m3.2-7.37l3.2-7.36m3.2-7.36l3.2-7.36m3.2-7.37l3.22-7.36m3.2-7.36l3.2-7.36m3.2-7.36l3.2-7.36m3.2-7.36l3.2-7.37m3.2-7.37l3.2-7.36m3.2-7.37l3.2-7.36m3.2-7.36l3.2-7.36m3.2-7.37l3.2-7.36m3.2-7.37l3.22-7.37m3.2-7.36l3.2-7.36m3.2-7.37l3.2-7.36m3.2-7.35l1.6-3.68" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4378.7 676.53l1.43-14.93 8.5 3.7zM-4251.3 383.47l-1.43 14.93-8.5-3.7z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4269.67 669.65l3.36-2.12m6.72-4.23l6.73-4.23m6.72-4.23l6.73-4.23m6.73-4.23l6.73-4.23m6.72-4.23l6.73-4.22m6.74-4.23l6.73-4.24m6.72-4.22l6.72-4.23m6.73-4.23l6.73-4.23m6.72-4.24l6.73-4.24m6.73-4.23l6.72-4.24m6.73-4.23l6.74-4.23m6.73-4.24l6.72-4.23m6.72-4.23l6.73-4.23m6.73-4.23l6.72-4.23m6.73-4.23l6.73-4.23m6.73-4.23l6.7-4.22m6.74-4.23l6.73-4.23m6.73-4.24l6.72-4.23m6.73-4.23l6.73-4.23m6.73-4.24l6.73-4.23m6.73-4.24l6.73-4.24m6.72-4.23l6.72-4.23m6.73-4.23l6.73-4.24m6.73-4.23l6.72-4.23m6.73-4.23l6.73-4.23m6.74-4.22l6.72-4.24m6.73-4.23l6.72-4.22m6.73-4.23l6.72-4.23m6.73-4.23l6.73-4.25m6.73-4.23l6.72-4.23m6.73-4.24l6.74-4.23m6.72-4.24l6.73-4.24m6.72-4.23l6.73-4.23m6.72-4.23l3.37-2.12" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4282.6 677.78l9.6-11.52 4.95 7.85zM-3812.74 382.22l-9.6 11.52-4.94-7.85z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4482.72 669.46l-3.37-2.18m-6.74-4.36l-6.74-4.35m-6.75-4.36l-6.75-4.35m-6.74-4.36l-6.75-4.37m-6.74-4.36l-6.76-4.35m-6.75-4.36l-6.75-4.36m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.74-4.35m-6.74-4.36l-6.75-4.35m-6.74-4.36l-6.75-4.37m-6.75-4.35l-6.74-4.36m-6.76-4.36l-6.74-4.36m-6.75-4.36l-6.75-4.35m-6.74-4.37l-6.75-4.36m-6.73-4.36l-6.75-4.35m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.76-4.36m-6.74-4.35l-6.75-4.36m-6.74-4.37l-6.74-4.36m-6.74-4.36l-6.75-4.34m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.74-4.36m-6.76-4.35l-6.75-4.36m-6.75-4.35l-6.75-4.36m-6.74-4.37l-6.75-4.36m-6.74-4.35l-6.74-4.36m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.74-4.35m-6.75-4.36l-6.76-4.36m-6.74-4.35l-6.75-4.36m-6.74-4.37l-6.74-4.35m-6.75-4.36l-3.36-2.18" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4469.9 677.74l-14.5-3.84 5.03-7.8zM-4927.28 382.26l14.5 3.84-5.03 7.8z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4797.85 662.74l-1.72-3.68m-3.45-7.38l-3.46-7.37m-3.45-7.37l-3.45-7.37m-3.46-7.38l-3.45-7.37m-3.44-7.37l-3.46-7.37m-3.45-7.38l-3.45-7.37m-3.45-7.37l-3.46-7.37m-3.46-7.37l-3.45-7.4m-3.47-7.36l-3.45-7.38m-3.45-7.37l-3.46-7.4m-3.45-7.36l-3.45-7.38m-3.46-7.37l-3.44-7.4m-3.45-7.36l-3.44-7.38m-3.46-7.37l-3.45-7.38m-3.45-7.38l-3.46-7.37m-3.45-7.38l-3.46-7.38m-3.46-7.38l-3.45-7.37m-3.45-7.38l-3.45-7.38m-3.46-7.38l-1.72-3.68" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4791.37 676.57l-10.25-10.96 8.4-3.92zM-4928.63 383.43l10.25 10.95-8.4 3.94z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4545.44 673.74l-3.65-1.66m-7.3-3.32l-7.3-3.32m-7.28-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.33m-7.3-3.3l-7.3-3.33m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.3m-7.3-3.33l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.3l-7.3-3.33m-7.3-3.32l-7.3-3.33m-7.3-3.3l-7.3-3.34m-7.28-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.33m-7.3-3.33l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.3l-7.3-3.33m-7.3-3.32l-7.28-3.32m-7.3-3.32l-7.3-3.3m-7.3-3.33l-7.3-3.32m-7.3-3.32l-7.3-3.33m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.33l-7.28-3.33m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.3m-7.3-3.33l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.3l-7.3-3.33m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-3.64-1.66" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4531.54 680.06l-14.9-1.7 3.83-8.42zM-5187.05 381.84l14.9 1.7-3.84 8.43z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4884.5 666.85l-2.92-2.74m-5.84-5.46l-5.84-5.48m-5.84-5.47l-5.84-5.5m-5.84-5.46l-5.84-5.47m-5.84-5.48l-5.84-5.48m-5.84-5.48l-5.84-5.47m-5.84-5.47l-5.84-5.48m-5.84-5.47l-5.84-5.47m-5.84-5.48l-5.84-5.47m-5.84-5.48l-5.84-5.47m-5.84-5.47l-5.84-5.47m-5.84-5.47l-5.84-5.48m-5.84-5.47l-5.84-5.46m-5.84-5.48l-5.84-5.47m-5.84-5.48l-5.84-5.48m-5.84-5.47l-5.84-5.48m-5.84-5.48l-5.84-5.47m-5.84-5.48l-5.84-5.47m-5.84-5.48l-5.84-5.47m-5.84-5.47l-5.84-5.48m-5.84-5.47l-5.84-5.48m-5.84-5.47l-5.84-5.47m-5.84-5.47l-5.84-5.47m-5.84-5.47l-5.84-5.5m-5.84-5.46l-5.84-5.48m-5.84-5.47l-2.92-2.75" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4873.36 677.3l-13.58-6.4 6.34-6.75zM-5187.64 382.7l13.58 6.4-6.34 6.75z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4214.56 673.74l3.65-1.66m7.3-3.32l7.3-3.32m7.28-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.33m7.3-3.3l7.3-3.33m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.3m7.3-3.33l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.3l7.3-3.33m7.3-3.32l7.3-3.33m7.3-3.3l7.3-3.34m7.28-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.33m7.3-3.33l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.3l7.3-3.33m7.3-3.32l7.28-3.32m7.3-3.32l7.3-3.3m7.3-3.33l7.3-3.32m7.3-3.32l7.3-3.33m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.33l7.3-3.33m7.28-3.32l7.3-3.32m7.3-3.32l7.3-3.3m7.3-3.33l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.3l7.3-3.33m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l3.64-1.66" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4228.46 680.06l11.07-10.12 3.85 8.43zM-3572.95 381.84l-11.06 10.13-3.85-8.44z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4250 241V125a6 6 0 0 1 6-6h542.34a6 6 0 0 1 6 6v88.38" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4250 256.26l-4.64-14.26h9.28z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-3695.66 213.36v1.02M-4380 103.17v37.33" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4380 87.9l4.64 14.27h-9.28zM-4380 155.76l-4.64-14.26h9.28z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4500 46a6 6 0 0 1 6-6h228a6 6 0 0 1 6 6v32.67a6 6 0 0 1-6 6h-228a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#r" transform="matrix(1,0,0,1,-4495,45) translate(26.929012345679013 25.29861111111111)"/><use xlink:href="#s" transform="matrix(1,0,0,1,-4495,45) translate(108.54012345679013 25.29861111111111)"/><path d="M-3960.27 663.2c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm1.88-3.52c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.52c0 .56-.44 1-1 1-.55 0-1-.44-1-1 0-.55.45-1 1-1 .56 0 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.52c0 .56-.44 1-1 1-.54 0-1-.44-1-1 0-.55.46-1 1-1 .56 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm1.86-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.52c0 .55-.44 1-1 1-.54 0-1-.45-1-1s.46-1 1-1c.56 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm1.86-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.52c0 .56-.43 1-1 1-.54 0-1-.44-1-1 0-.55.46-1 1-1 .57 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm1.86-3.5c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm1.87-3.5c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.52c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.57.45-1 1-1 .56 0 1 .43 1 1zm1.88-3.52c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.87-3.5c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.57.45-1 1-1 .56 0 1 .43 1 1zm1.88-3.52c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.57.44-1 1-1 .55 0 1 .43 1 1zm1.87-3.52c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm1.87-3.52c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm1.88-3.52c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm1.88-3.52c0 .55-.44 1-1 1-.54 0-1-.45-1-1s.46-1 1-1c.56 0 1 .45 1 1zm1.88-3.5c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.57.44-1 1-1 .55 0 1 .43 1 1zm1.87-3.52c0 .55-.44 1-1 1-.54 0-1-.45-1-1s.46-1 1-1c.56 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.57 0-1-.45-1-1 0-.56.43-1 1-1 .54 0 1 .44 1 1zm1.86-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.52c0 .55-.43 1-1 1-.54 0-1-.45-1-1s.46-1 1-1c.57 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm1.86-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm1.87-3.5c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1z" fill="#3a414a"/><path d="M-3968.47 676.65l2.64-14.77 8.17 4.38zM-3811.53 383.35l-2.64 14.77-8.17-4.38z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-3850.03 666.3c0 .56-.45 1-1 1s-1-.44-1-1c0-.54.45-1 1-1s1 .46 1 1zm2.8-2.83c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm2.8-2.85c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .56-.44 1-1 1-.55 0-1-.44-1-1 0-.54.45-1 1-1 .56 0 1 .46 1 1zm2.8-2.83c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .54-.44 1-1 1-.54 0-1-.46-1-1 0-.56.46-1 1-1 .56 0 1 .44 1 1zm2.8-2.85c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .55 0 1 .46 1 1zm2.8-2.83c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .54-.46 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.85c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .55 0 1 .46 1 1zm2.8-2.83c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.85c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .57-.45 1-1 1-.56 0-1-.43-1-1 0-.54.44-1 1-1 .55 0 1 .46 1 1zm2.8-2.83c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.85c0 .56-.46 1-1 1-.57 0-1-.44-1-1 0-.55.43-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .57-.45 1-1 1s-1-.43-1-1c0-.54.45-1 1-1s1 .46 1 1zm2.8-2.83c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.57 0-1-.45-1-1 0-.56.43-1 1-1 .54 0 1 .44 1 1zm2.78-2.85c0 .57-.45 1-1 1s-1-.43-1-1c0-.54.45-1 1-1s1 .46 1 1zm2.8-2.82c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.84c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.78-2.84c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.57 0-1-.44-1-1 0-.55.43-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .56-.44 1-1 1-.55 0-1-.44-1-1 0-.55.45-1 1-1 .56 0 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm2.8-2.83c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.78-2.85c0 .56-.44 1-1 1-.55 0-1-.44-1-1 0-.55.45-1 1-1 .56 0 1 .45 1 1zm2.8-2.83c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.84c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm2.8-2.85c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.84c0 .56-.46 1-1 1-.57 0-1-.44-1-1 0-.55.43-1 1-1 .54 0 1 .45 1 1zm2.78-2.83c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.84c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.85c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.57 0-1-.44-1-1 0-.54.43-1 1-1 .54 0 1 .46 1 1zm2.78-2.83c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.85c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .54 0 1 .46 1 1zm2.78-2.83c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.85c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .54 0 1 .46 1 1zm2.78-2.83c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .54-.46 1-1 1-.57 0-1-.46-1-1 0-.56.43-1 1-1 .54 0 1 .44 1 1zm2.78-2.85c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .54 0 1 .46 1 1zm2.78-2.83c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1z" fill="#3a414a"/><path d="M-3861.73 677.2l6.7-13.43 6.6 6.5zM-3572.27 382.8l-6.7 13.43-6.6-6.5z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-3658.5 775h30.26M-3658.46 775h-1.54" stroke="#3a414a" stroke-width="3" fill="none"/><path d="M-3612.47 775l-14.27 4.63v-9.26z" stroke="#3a414a" stroke-width="3" fill="#3a414a"/><path d="M-3658.5 815h7.57m15.13 0h7.56M-3658.46 815h-1.54" stroke="#3a414a" stroke-width="3" fill="none"/><path d="M-3612.47 815l-14.27 4.63v-9.26z" stroke="#3a414a" stroke-width="3" fill="#3a414a"/><path d="M-3657 848.33c0 .83-.67 1.5-1.5 1.5s-1.5-.67-1.5-1.5.67-1.5 1.5-1.5 1.5.67 1.5 1.5zm6.05 0c0 .83-.67 1.5-1.5 1.5s-1.5-.67-1.5-1.5.67-1.5 1.5-1.5 1.5.67 1.5 1.5zm6.05 0c0 .83-.67 1.5-1.5 1.5-.82 0-1.5-.67-1.5-1.5s.68-1.5 1.5-1.5c.83 0 1.5.67 1.5 1.5zm6.06 0c0 .83-.67 1.5-1.5 1.5s-1.5-.67-1.5-1.5.67-1.5 1.5-1.5 1.5.67 1.5 1.5zm6.05 0c0 .83-.66 1.5-1.5 1.5-.82 0-1.5-.67-1.5-1.5s.68-1.5 1.5-1.5c.84 0 1.5.67 1.5 1.5z" fill="#3a414a"/><path d="M-3658.46 848.33h-1.54" stroke="#3a414a" stroke-width="3" fill="none"/><path d="M-3612.47 848.33l-14.27 4.64v-9.27z" stroke="#3a414a" stroke-width="3" fill="#3a414a"/><path d="M-3607.62 766a6 6 0 0 1 6-6h92.76a6 6 0 0 1 6 6v18a6 6 0 0 1-6 6h-92.76a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#t" transform="matrix(1,0,0,1,-3602.6190476190477,765) translate(0 19.555555555555554)"/><path d="M-3607.62 839.33a6 6 0 0 1 6-6H-3446a6 6 0 0 1 6 6v18a6 6 0 0 1-6 6h-155.62a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#u" transform="matrix(1,0,0,1,-3602.6190476190477,838.3333333333334) translate(0 19.555555555555554)"/><path d="M-3607.62 806a6 6 0 0 1 6-6h113.7a6 6 0 0 1 6 6v18a6 6 0 0 1-6 6h-113.7a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#v" transform="matrix(1,0,0,1,-3602.6190476190473,805) translate(0 19.555555555555554)"/><path d="M-3694.76 552a12 12 0 0 1 12-12h276a12 12 0 0 1 12 12v198.96a12 12 0 0 1-12 12h-276a12 12 0 0 1-12-12z" stroke="#000" stroke-opacity="0" stroke-width="2" fill-opacity="0"/><path d="M-3638.76 613.6c0 6.63-5.37 12-12 12s-12-5.37-12-12 5.37-12 12-12 12 5.37 12 12z" stroke="#000" stroke-opacity=".25" fill="#cfe4ff"/><path d="M-3638.76 651.2c0 6.63-5.37 12-12 12s-12-5.37-12-12 5.37-12 12-12 12 5.37 12 12z" stroke="#000" stroke-opacity=".25" fill="#ffdda6"/><path d="M-3638.76 688.8c0 6.63-5.37 12-12 12s-12-5.37-12-12 5.37-12 12-12 12 5.37 12 12z" stroke="#000" stroke-opacity=".25" fill="#dedeff"/><use xlink:href="#w" transform="matrix(1,0,0,1,-3662.7619047619046,564) translate(0 19.933333333333334)"/><g><use xlink:href="#x" transform="matrix(1,0,0,1,-3626.7619047619046,639.2) translate(0 19.933333333333334)"/><use xlink:href="#y" transform="matrix(1,0,0,1,-3626.7619047619046,639.2) translate(65.57222222222221 19.933333333333334)"/></g><g><use xlink:href="#z" transform="matrix(1,0,0,1,-3626.7619047619046,676.8) translate(0 20.153333333333336)"/><use xlink:href="#A" transform="matrix(1,0,0,1,-3626.7619047619046,676.8) translate(0 52.71333333333334)"/></g><g><use xlink:href="#x" transform="matrix(1,0,0,1,-3626.7619047619046,601.6) translate(0 19.933333333333334)"/><use xlink:href="#B" transform="matrix(1,0,0,1,-3626.7619047619046,601.6) translate(65.57222222222221 19.933333333333334)"/></g><path d="M-3810 241v-20.62a6 6 0 0 1 6-6h228a6 6 0 0 1 6 6V241" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-3810 256.26l-4.64-14.26h9.28zM-3570 256.26l-4.64-14.26h9.28z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><defs><path d="M212-179c-10-28-35-45-73-45-59 0-87 40-87 99 0 60 29 101 89 101 43 0 62-24 78-52l27 14C228-24 195 4 139 4 59 4 22-46 18-125c-6-104 99-153 187-111 19 9 31 26 39 46" id="C"/><path d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="D"/><path d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="E"/><path d="M85-194c31 0 48 13 60 33l-1-100h32l1 261h-30c-2-10 0-23-3-31C134-8 116 4 85 4 32 4 16-35 15-94c0-66 23-100 70-100zm9 24c-40 0-46 34-46 75 0 40 6 74 45 74 42 0 51-32 51-76 0-42-9-74-50-73" id="F"/><path d="M24-231v-30h32v30H24zM24 0v-190h32V0H24" id="G"/><path d="M117-194c89-4 53 116 60 194h-32v-121c0-31-8-49-39-48C34-167 62-67 57 0H25l-1-190h30c1 10-1 24 2 32 11-22 29-35 61-36" id="H"/><path d="M141-36C126-15 110 5 73 4 37 3 15-17 15-53c-1-64 63-63 125-63 3-35-9-54-41-54-24 1-41 7-42 31l-33-3c5-37 33-52 76-52 45 0 72 20 72 64v82c-1 20 7 32 28 27v20c-31 9-61-2-59-35zM48-53c0 20 12 33 32 33 41-3 63-29 60-74-43 2-92-5-92 41" id="I"/><path d="M59-47c-2 24 18 29 38 22v24C64 9 27 4 27-40v-127H5v-23h24l9-43h21v43h35v23H59v120" id="J"/><g id="a"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#C"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.83333333333333,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,52.8827160493827,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,62.43209876543209,0)" xlink:href="#F"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,78.48148148148147,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.82098765432096,0)" xlink:href="#H"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,100.87037037037035,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,116.91975308641973,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,124.94444444444444,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,140.99382716049382,0)" xlink:href="#E"/></g><path d="M140-251c81 0 123 46 123 126C263-46 219 4 140 4 59 4 17-45 17-125s42-126 123-126zm0 227c63 0 89-41 89-101s-29-99-89-99c-61 0-89 39-89 99S79-25 140-24" id="K"/><path d="M108 0H70L1-190h34L89-25l56-165h34" id="L"/><path d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="M"/><path d="M24 0v-261h32V0H24" id="N"/><g id="b"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#K"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,22.469135802469136,0)" xlink:href="#L"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.91358024691358,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,52.96296296296296,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,62.51234567901234,0)" xlink:href="#N"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,68.85185185185185,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.90123456790123,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,94.4506172839506,0)" xlink:href="#F"/></g><path d="M240 0l2-218c-23 76-54 145-80 218h-23L58-218 59 0H30v-248h44l77 211c21-75 51-140 76-211h43V0h-30" id="O"/><path d="M177-190C167-65 218 103 67 71c-23-6-38-20-44-43l32-5c15 47 100 32 89-28v-30C133-14 115 1 83 1 29 1 15-40 15-95c0-56 16-97 71-98 29-1 48 16 59 35 1-10 0-23 2-32h30zM94-22c36 0 50-32 50-73 0-42-14-75-50-75-39 0-46 34-46 75s6 73 46 73" id="P"/><g id="c"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#O"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,23.993827160493826,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,30.333333333333332,0)" xlink:href="#F"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,46.382716049382715,0)" xlink:href="#F"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,62.4320987654321,0)" xlink:href="#N"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,68.77160493827161,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.82098765432099,0)" xlink:href="#O"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,108.81481481481481,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,124.86419753086419,0)" xlink:href="#H"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,140.91358024691357,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,156.96296296296296,0)" xlink:href="#P"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,173.01234567901238,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,189.06172839506175,0)" xlink:href="#E"/></g><path d="M160-131c35 5 61 23 61 61C221 17 115-2 30 0v-248c76 3 177-17 177 60 0 33-19 50-47 57zm-97-11c50-1 110 9 110-42 0-47-63-36-110-37v79zm0 115c55-2 124 14 124-45 0-56-70-42-124-44v89" id="Q"/><path d="M143 0L79-87 56-68V0H24v-261h32v163l83-92h37l-77 82L181 0h-38" id="R"/><g id="d"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#Q"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,19.25925925925926,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,28.80864197530864,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,44.858024691358025,0)" xlink:href="#R"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,59.30246913580247,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,75.35185185185185,0)" xlink:href="#E"/></g><path d="M233-177c-1 41-23 64-60 70L243 0h-38l-65-103H63V0H30v-248c88 3 205-21 203 71zM63-129c60-2 137 13 137-47 0-61-80-42-137-45v92" id="S"/><path d="M84 4C-5 8 30-112 23-190h32v120c0 31 7 50 39 49 72-2 45-101 50-169h31l1 190h-30c-1-10 1-25-2-33-11 22-28 36-60 37" id="T"/><g id="e"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#S"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.83333333333333,0)" xlink:href="#T"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,52.8827160493827,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,60.9074074074074,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,76.95679012345678,0)" xlink:href="#E"/></g><path d="M197 0v-115H63V0H30v-248h33v105h134v-105h34V0h-34" id="U"/><path d="M135-143c-3-34-86-38-87 0 15 53 115 12 119 90S17 21 10-45l28-5c4 36 97 45 98 0-10-56-113-15-118-90-4-57 82-63 122-42 12 7 21 19 24 35" id="V"/><path d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="W"/><g id="f"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#U"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,27.123456790123456,0)" xlink:href="#V"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,41.5679012345679,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,49.59259259259259,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,65.64197530864197,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,75.19135802469135,0)" xlink:href="#W"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,89.6358024691358,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,95.9753086419753,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,112.02469135802468,0)" xlink:href="#N"/></g><path d="M230 0l2-204L168 0h-37L68-204 70 0H24v-248h70l56 185 57-185h69V0h-46" id="X"/><path d="M133-34C117-15 103 5 69 4 32 3 11-16 11-54c-1-60 55-63 116-61 1-26-3-47-28-47-18 1-26 9-28 27l-52-2c7-38 36-58 82-57s74 22 75 68l1 82c-1 14 12 18 25 15v27c-30 8-71 5-69-32zm-48 3c29 0 43-24 42-57-32 0-66-3-65 30 0 17 8 27 23 27" id="Y"/><path d="M137-138c1-29-70-34-71-4 15 46 118 7 119 86 1 83-164 76-172 9l43-7c4 19 20 25 44 25 33 8 57-30 24-41C81-84 22-81 20-136c-2-80 154-74 161-7" id="Z"/><path d="M115-3C79 11 28 4 28-45v-112H4v-33h27l15-45h31v45h36v33H77v99c-1 23 16 31 38 25v30" id="aa"/><path d="M185-48c-13 30-37 53-82 52C43 2 14-33 14-96s30-98 90-98c62 0 83 45 84 108H66c0 31 8 55 39 56 18 0 30-7 34-22zm-45-69c5-46-57-63-70-21-2 6-4 13-4 21h74" id="ab"/><path d="M135-150c-39-12-60 13-60 57V0H25l-1-190h47c2 13-1 29 3 40 6-28 27-53 61-41v41" id="ac"/><g id="g"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#X"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,23.993827160493826,0)" xlink:href="#Y"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,40.04320987654321,0)" xlink:href="#Z"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,56.092592592592595,0)" xlink:href="#aa"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,65.64197530864197,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,81.69135802469135,0)" xlink:href="#ac"/></g><path d="M128 0H69L1-190h53L99-40l48-150h52" id="ad"/><g id="h"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#Z"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,16.049382716049383,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,32.098765432098766,0)" xlink:href="#ac"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,43.33333333333334,0)" xlink:href="#ad"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,59.38271604938272,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,75.4320987654321,0)" xlink:href="#ac"/></g><path d="M140-251c80 0 125 45 125 126 0 70-33 111-92 124 6 28 34 38 68 31l-1 36C177 83 129 55 120 2 51-6 15-50 15-125c0-81 44-126 125-126zm-1 214c52 0 73-35 73-88 0-50-21-86-72-86-52 0-73 35-73 86s22 88 72 88" id="ae"/><path d="M85 4C-2 5 27-109 22-190h50c7 57-23 150 33 157 60-5 35-97 40-157h50l1 190h-47c-2-12 1-28-3-38-12 25-28 42-61 42" id="af"/><path d="M123 10C108 53 80 86 19 72V37c35 8 53-11 59-39L3-190h52l48 148c12-52 28-100 44-148h51" id="ag"/><g id="i"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#ae"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,22.469135802469136,0)" xlink:href="#af"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,40.04320987654321,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,56.092592592592595,0)" xlink:href="#ac"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,67.32716049382717,0)" xlink:href="#ag"/></g><path d="M24-248c120-7 223 5 221 122C244-46 201 0 124 0H24v-248zM76-40c74 7 117-18 117-86 0-67-45-88-117-82v168" id="ah"/><g id="j"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#ah"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#Y"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.83333333333333,0)" xlink:href="#aa"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,46.3827160493827,0)" xlink:href="#Y"/></g><path fill="#3a414a" d="M30-248c87 1 191-15 191 75 0 78-77 80-158 76V0H30v-248zm33 125c57 0 124 11 124-50 0-59-68-47-124-48v98" id="ai"/><path fill="#3a414a" d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="aj"/><path fill="#3a414a" d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="ak"/><path fill="#3a414a" d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="al"/><path fill="#3a414a" d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="am"/><path fill="#3a414a" d="M135-143c-3-34-86-38-87 0 15 53 115 12 119 90S17 21 10-45l28-5c4 36 97 45 98 0-10-56-113-15-118-90-4-57 82-63 122-42 12 7 21 19 24 35" id="an"/><g id="k"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#ai"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,14.814814814814813,0)" xlink:href="#aj"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,22.160493827160494,0)" xlink:href="#ak"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,34.50617283950617,0)" xlink:href="#al"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,45.61728395061728,0)" xlink:href="#am"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,57.962962962962955,0)" xlink:href="#an"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,69.07407407407406,0)" xlink:href="#an"/></g><path d="M209 0H11v-25l151-195H24v-28h176v25L50-27h159V0" id="ao"/><path d="M194 0L95-120 63-95V0H30v-248h33v124l119-124h40L117-140 236 0h-42" id="ap"/><path d="M115-194c55 1 70 41 70 98S169 2 115 4C84 4 66-9 55-30l1 105H24l-1-265h31l2 30c10-21 28-34 59-34zm-8 174c40 0 45-34 45-75s-6-73-45-74c-42 0-51 32-51 76 0 43 10 73 51 73" id="aq"/><g id="l"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#ao"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,17.574074074074073,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,33.623456790123456,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,49.672839506172835,0)" xlink:href="#ap"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,68.9320987654321,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.98148148148148,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,101.03086419753086,0)" xlink:href="#aq"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,117.08024691358024,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,133.12962962962962,0)" xlink:href="#E"/></g><path d="M30-248c118-7 216 8 213 122C240-48 200 0 122 0H30v-248zM63-27c89 8 146-16 146-99s-60-101-146-95v194" id="ar"/><g id="m"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#ar"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.83333333333333,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,52.8827160493827,0)" xlink:href="#aq"/></g><g id="n"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#V"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,14.444444444444443,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,22.469135802469136,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,38.51851851851852,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,48.0679012345679,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,64.11728395061728,0)" xlink:href="#P"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,80.16666666666666,0)" xlink:href="#M"/></g><g id="o"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#O"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,23.993827160493826,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,40.04320987654321,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,48.0679012345679,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,64.11728395061728,0)" xlink:href="#F"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,80.16666666666666,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,96.21604938271605,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,104.24074074074073,0)" xlink:href="#I"/></g><path d="M24 0v-248h195v40H76v63h132v40H76v65h150V0H24" id="as"/><path d="M144 0l-44-69L55 0H2l70-98-66-92h53l41 62 40-62h54l-67 91 71 99h-54" id="at"/><path d="M135-194c87-1 58 113 63 194h-50c-7-57 23-157-34-157-59 0-34 97-39 157H25l-1-190h47c2 12-1 28 3 38 12-26 28-41 61-42" id="au"/><path d="M25 0v-261h50V0H25" id="av"/><g id="p"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#as"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,19.25925925925926,0)" xlink:href="#at"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,35.308641975308646,0)" xlink:href="#aa"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,44.858024691358025,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,60.907407407407405,0)" xlink:href="#ac"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,72.14197530864197,0)" xlink:href="#au"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,89.71604938271605,0)" xlink:href="#Y"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,105.76543209876543,0)" xlink:href="#av"/></g><path d="M88-194c31-1 46 15 58 34l-1-101h50l1 261h-48c-2-10 0-23-3-31C134-8 116 4 84 4 32 4 16-41 15-95c0-56 19-97 73-99zm17 164c33 0 40-30 41-66 1-37-9-64-41-64s-38 30-39 65c0 43 13 65 39 65" id="aw"/><path d="M135-194c53 0 70 44 70 98 0 56-19 98-73 100-31 1-45-17-59-34 3 33 2 69 2 105H25l-1-265h48c2 10 0 23 3 31 11-24 29-35 60-35zM114-30c33 0 39-31 40-66 0-38-9-64-40-64-56 0-55 130 0 130" id="ax"/><path d="M190-63c-7 42-38 67-86 67-59 0-84-38-90-98-12-110 154-137 174-36l-49 2c-2-19-15-32-35-32-30 0-35 28-38 64-6 74 65 87 74 30" id="ay"/><path d="M25-224v-37h50v37H25zM25 0v-190h50V0H25" id="az"/><g id="q"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#aw"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,17.574074074074073,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,33.623456790123456,0)" xlink:href="#ax"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,51.197530864197525,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,67.24691358024691,0)" xlink:href="#au"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.82098765432099,0)" xlink:href="#aw"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,102.39506172839505,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,118.44444444444444,0)" xlink:href="#au"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,136.0185185185185,0)" xlink:href="#ay"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,152.0679012345679,0)" xlink:href="#az"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,160.09259259259258,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,176.14197530864197,0)" xlink:href="#Z"/></g><g id="r"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#C"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#N"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,27.123456790123456,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,33.46296296296296,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,49.51234567901234,0)" xlink:href="#H"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,65.56172839506172,0)" xlink:href="#J"/></g><path d="M145-31C134-9 116 4 85 4 32 4 16-35 15-94c0-59 17-99 70-100 32-1 48 14 60 33 0-11-1-24 2-32h30l-1 268h-32zM93-21c41 0 51-33 51-76s-8-73-50-73c-40 0-46 35-46 75s5 74 45 74" id="aA"/><g id="s"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#aA"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,16.049382716049383,0)" xlink:href="#T"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,32.098765432098766,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,48.14814814814815,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,57.697530864197525,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,64.03703703703704,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,80.08641975308642,0)" xlink:href="#V"/></g><path d="M140-251c81 0 123 46 123 126C263-53 228-8 163 1c7 30 30 48 69 40v23c-55 16-95-15-103-61C56-3 17-48 17-125c0-80 42-126 123-126zm0 227c63 0 89-41 89-101s-29-99-89-99c-61 0-89 39-89 99S79-25 140-24" id="aB"/><g id="t"><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,0,0)" xlink:href="#aB"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,19.01234567901234,0)" xlink:href="#T"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,32.59259259259258,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,46.17283950617282,0)" xlink:href="#E"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,54.25308641975307,0)" xlink:href="#G"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,59.617283950617264,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,73.1975308641975,0)" xlink:href="#V"/></g><path d="M0 4l72-265h28L28 4H0" id="aC"/><path d="M185-189c-5-48-123-54-124 2 14 75 158 14 163 119 3 78-121 87-175 55-17-10-28-26-33-46l33-7c5 56 141 63 141-1 0-78-155-14-162-118-5-82 145-84 179-34 5 7 8 16 11 25" id="aD"/><path d="M210-169c-67 3-38 105-44 169h-31v-121c0-29-5-50-35-48C34-165 62-65 56 0H25l-1-190h30c1 10-1 24 2 32 10-44 99-50 107 0 11-21 27-35 58-36 85-2 47 119 55 194h-31v-121c0-29-5-49-35-48" id="aE"/><g id="u"><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,0,0)" xlink:href="#ar"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,17.586419753086414,0)" xlink:href="#I"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,31.166666666666657,0)" xlink:href="#J"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,37.95679012345678,0)" xlink:href="#I"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,51.537037037037024,0)" xlink:href="#aC"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,58.327160493827144,0)" xlink:href="#aD"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,74.62345679012344,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,88.20370370370368,0)" xlink:href="#P"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,101.78395061728392,0)" xlink:href="#aE"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,122.08641975308639,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,135.66666666666663,0)" xlink:href="#H"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,149.24691358024688,0)" xlink:href="#J"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,156.037037037037,0)" xlink:href="#V"/></g><g id="v"><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,0,0)" xlink:href="#O"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,20.302469135802465,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,33.88271604938271,0)" xlink:href="#J"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,40.67283950617283,0)" xlink:href="#I"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,54.253086419753075,0)" xlink:href="#F"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,67.83333333333331,0)" xlink:href="#I"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,81.41358024691355,0)" xlink:href="#J"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,88.20370370370368,0)" xlink:href="#I"/></g><g id="x"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#ar"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,17.746296296296293,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,25.899999999999995,0)" xlink:href="#T"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,39.603703703703694,0)" xlink:href="#G"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,45.01666666666666,0)" xlink:href="#F"/></g><g id="y"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#V"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,12.33333333333333,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,26.03703703703703,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,34.190740740740736,0)" xlink:href="#L"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,46.524074074074065,0)" xlink:href="#G"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,51.93703703703703,0)" xlink:href="#W"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,64.27037037037036,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,77.97407407407405,0)" xlink:href="#V"/></g><path d="M30 0v-248h187v28H63v79h144v27H63v87h162V0H30" id="aF"/><path d="M141 0L90-78 38 0H4l68-98-65-92h35l48 74 47-74h35l-64 92 68 98h-35" id="aG"/><g id="z"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#aF"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,16.444444444444443,0)" xlink:href="#aG"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,28.77777777777777,0)" xlink:href="#J"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,35.62962962962961,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,49.33333333333332,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,57.48703703703703,0)" xlink:href="#H"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,71.19074074074072,0)" xlink:href="#I"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,84.89444444444442,0)" xlink:href="#N"/></g><g id="A"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#F"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,13.7037037037037,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,27.4074074074074,0)" xlink:href="#aq"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,41.11111111111111,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,54.8148148148148,0)" xlink:href="#H"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,68.5185185185185,0)" xlink:href="#F"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,82.22222222222221,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,95.9259259259259,0)" xlink:href="#H"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,109.62962962962959,0)" xlink:href="#G"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,115.04259259259256,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,128.74629629629626,0)" xlink:href="#V"/></g><g id="B"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#V"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,12.33333333333333,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,26.03703703703703,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,34.190740740740736,0)" xlink:href="#L"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,46.524074074074065,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,60.22777777777777,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,68.38148148148147,0)" xlink:href="#V"/></g></defs></g></svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:lucid="lucid" width="1967.24" height="872.83"><g transform="translate(5341 -19.5)" lucid:page-tab-id="0_0"><path d="M-5320 166a6 6 0 0 1 6-6h508a6 6 0 0 1 6 6v308a6 6 0 0 1-6 6h-508a6 6 0 0 1-6-6zM-4640 166a6 6 0 0 1 6-6h508a6 6 0 0 1 6 6v308a6 6 0 0 1-6 6h-508a6 6 0 0 1-6-6zM-3960 166a6 6 0 0 1 6-6h508a6 6 0 0 1 6 6v308a6 6 0 0 1-6 6h-508a6 6 0 0 1-6-6z" stroke="#cfe4ff" stroke-width="2" fill="#cfe4ff"/><path d="M-5280 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#a" transform="matrix(1,0,0,1,-5275,265) translate(9.728395061728392 60.923611111111114)"/><path d="M-5020 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#b" transform="matrix(1,0,0,1,-5015,265) translate(29.750000000000007 60.923611111111114)"/><path d="M-3920 266a6 6 0 0 1 6-6h208a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-208a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#c" transform="matrix(1,0,0,1,-3915,265) translate(5.694444444444429 60.923611111111114)"/><path d="M-4340 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#d" transform="matrix(1,0,0,1,-4335,265) translate(42.54938271604939 60.923611111111114)"/><path d="M-4600 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#e" transform="matrix(1,0,0,1,-4595,265) translate(41.74691358024692 60.923611111111114)"/><path d="M-3660 266a6 6 0 0 1 6-6h168a6 6 0 0 1 6 6v108a6 6 0 0 1-6 6h-168a6 6 0 0 1-6-6z" stroke="#fc9432" fill="#ffdda6"/><use xlink:href="#f" transform="matrix(1,0,0,1,-3655,265) translate(25.817901234567906 60.923611111111114)"/><path d="M-4401 320h42" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4416.26 320l14.26-4.63v9.26zM-4343.74 320l-14.26 4.63v-9.26z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-5320 156.33a6 6 0 0 1 6-6h208a6 6 0 0 1 6 6v67.34a6 6 0 0 1-6 6h-208a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#g" transform="matrix(1,0,0,1,-5315,155.33333333333334) translate(11.191358024691354 40.611111111111114)"/><use xlink:href="#h" transform="matrix(1,0,0,1,-5315,155.33333333333334) translate(112.14197530864197 40.611111111111114)"/><path d="M-4640 166a6 6 0 0 1 6-6h188a6 6 0 0 1 6 6v48a6 6 0 0 1-6 6h-188a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#i" transform="matrix(1,0,0,1,-4635,165.00000000000006) translate(5.966049382716051 29.36111111111111)"/><use xlink:href="#h" transform="matrix(1,0,0,1,-4635,165.00000000000006) translate(97.36728395061729 29.36111111111111)"/><path d="M-3960 166a6 6 0 0 1 6-6h188a6 6 0 0 1 6 6v48a6 6 0 0 1-6 6h-188a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#j" transform="matrix(1,0,0,1,-3955,165.00000000000006) translate(16.43827160493828 29.36111111111111)"/><use xlink:href="#h" transform="matrix(1,0,0,1,-3955,165.00000000000006) translate(86.89506172839506 29.36111111111111)"/><path d="M-5000 646a6 6 0 0 1 6-6h1228a6 6 0 0 1 6 6v188a6 6 0 0 1-6 6h-1228a6 6 0 0 1-6-6z" stroke="#ced4db" fill="#f2f3f5"/><use xlink:href="#k" transform="matrix(1,0,0,1,-4988,652) translate(567.9074074074074 94.77777777777777)"/><path d="M-4530 686a6 6 0 0 1 6-6h288a6 6 0 0 1 6 6v68a6 6 0 0 1-6 6h-288a6 6 0 0 1-6-6z" stroke="#9391ff" fill="#dedeff"/><use xlink:href="#l" transform="matrix(1,0,0,1,-4518,692) translate(66.6604938271605 33.611111111111114)"/><path d="M-4120 686a6 6 0 0 1 6-6h288a6 6 0 0 1 6 6v68a6 6 0 0 1-6 6h-288a6 6 0 0 1-6-6z" stroke="#9391ff" fill="#dedeff"/><use xlink:href="#m" transform="matrix(1,0,0,1,-4108,692) translate(51.4135802469136 33.611111111111114)"/><use xlink:href="#n" transform="matrix(1,0,0,1,-4108,692) translate(128.37037037037038 33.611111111111114)"/><path d="M-4940 686a6 6 0 0 1 6-6h288a6 6 0 0 1 6 6v68a6 6 0 0 1-6 6h-288a6 6 0 0 1-6-6z" stroke="#9391ff" fill="#dedeff"/><use xlink:href="#o" transform="matrix(1,0,0,1,-4928,692) translate(25.734567901234584 33.611111111111114)"/><use xlink:href="#n" transform="matrix(1,0,0,1,-4928,692) translate(154.0493827160494 33.611111111111114)"/><path d="M-4560 766.67a6 6 0 0 1 6-6h348a6 6 0 0 1 6 6V834a6 6 0 0 1-6 6h-348a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#p" transform="matrix(1,0,0,1,-4555,765.6666666666666) translate(17.99691358024691 40.611111111111114)"/><use xlink:href="#q" transform="matrix(1,0,0,1,-4555,765.6666666666666) translate(139.81172839506172 40.611111111111114)"/><path d="M-4387.38 662.53l-1.6-3.68m-3.2-7.36l-3.2-7.38m-3.2-7.36l-3.2-7.36m-3.2-7.36l-3.2-7.37m-3.2-7.36l-3.2-7.35m-3.2-7.37l-3.2-7.36m-3.2-7.36l-3.2-7.36m-3.2-7.37l-3.22-7.36m-3.2-7.36l-3.2-7.36m-3.2-7.36l-3.2-7.36m-3.2-7.36l-3.2-7.37m-3.2-7.37l-3.2-7.36m-3.2-7.37l-3.2-7.36m-3.2-7.36l-3.2-7.36m-3.2-7.37l-3.2-7.36m-3.2-7.37l-3.22-7.37m-3.2-7.36l-3.2-7.36m-3.2-7.37l-3.2-7.36m-3.2-7.35l-1.6-3.68" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4381.3 676.53l-9.93-11.23 8.5-3.7zM-4508.7 383.47l9.93 11.23-8.5 3.7z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4372.62 662.53l1.6-3.68m3.2-7.36l3.2-7.38m3.2-7.36l3.2-7.36m3.2-7.36l3.2-7.37m3.2-7.36l3.2-7.35m3.2-7.37l3.2-7.36m3.2-7.36l3.2-7.36m3.2-7.37l3.22-7.36m3.2-7.36l3.2-7.36m3.2-7.36l3.2-7.36m3.2-7.36l3.2-7.37m3.2-7.37l3.2-7.36m3.2-7.37l3.2-7.36m3.2-7.36l3.2-7.36m3.2-7.37l3.2-7.36m3.2-7.37l3.22-7.37m3.2-7.36l3.2-7.36m3.2-7.37l3.2-7.36m3.2-7.35l1.6-3.68" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4378.7 676.53l1.43-14.93 8.5 3.7zM-4251.3 383.47l-1.43 14.93-8.5-3.7z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4269.67 669.65l3.36-2.12m6.72-4.23l6.73-4.23m6.72-4.23l6.73-4.23m6.73-4.23l6.73-4.23m6.72-4.23l6.73-4.22m6.74-4.23l6.73-4.24m6.72-4.22l6.72-4.23m6.73-4.23l6.73-4.23m6.72-4.24l6.73-4.24m6.73-4.23l6.72-4.24m6.73-4.23l6.74-4.23m6.73-4.24l6.72-4.23m6.72-4.23l6.73-4.23m6.73-4.23l6.72-4.23m6.73-4.23l6.73-4.23m6.73-4.23l6.7-4.22m6.74-4.23l6.73-4.23m6.73-4.24l6.72-4.23m6.73-4.23l6.73-4.23m6.73-4.24l6.73-4.23m6.73-4.24l6.73-4.24m6.72-4.23l6.72-4.23m6.73-4.23l6.73-4.24m6.73-4.23l6.72-4.23m6.73-4.23l6.73-4.23m6.74-4.22l6.72-4.24m6.73-4.23l6.72-4.22m6.73-4.23l6.72-4.23m6.73-4.23l6.73-4.25m6.73-4.23l6.72-4.23m6.73-4.24l6.74-4.23m6.72-4.24l6.73-4.24m6.72-4.23l6.73-4.23m6.72-4.23l3.37-2.12" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4282.6 677.78l9.6-11.52 4.95 7.85zM-3812.74 382.22l-9.6 11.52-4.94-7.85z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4482.72 669.46l-3.37-2.18m-6.74-4.36l-6.74-4.35m-6.75-4.36l-6.75-4.35m-6.74-4.36l-6.75-4.37m-6.74-4.36l-6.76-4.35m-6.75-4.36l-6.75-4.36m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.74-4.35m-6.74-4.36l-6.75-4.35m-6.74-4.36l-6.75-4.37m-6.75-4.35l-6.74-4.36m-6.76-4.36l-6.74-4.36m-6.75-4.36l-6.75-4.35m-6.74-4.37l-6.75-4.36m-6.73-4.36l-6.75-4.35m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.76-4.36m-6.74-4.35l-6.75-4.36m-6.74-4.37l-6.74-4.36m-6.74-4.36l-6.75-4.34m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.74-4.36m-6.76-4.35l-6.75-4.36m-6.75-4.35l-6.75-4.36m-6.74-4.37l-6.75-4.36m-6.74-4.35l-6.74-4.36m-6.75-4.36l-6.74-4.36m-6.75-4.36l-6.74-4.35m-6.75-4.36l-6.76-4.36m-6.74-4.35l-6.75-4.36m-6.74-4.37l-6.74-4.35m-6.75-4.36l-3.36-2.18" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4469.9 677.74l-14.5-3.84 5.03-7.8zM-4927.28 382.26l14.5 3.84-5.03 7.8z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4797.85 662.74l-1.72-3.68m-3.45-7.38l-3.46-7.37m-3.45-7.37l-3.45-7.37m-3.46-7.38l-3.45-7.37m-3.44-7.37l-3.46-7.37m-3.45-7.38l-3.45-7.37m-3.45-7.37l-3.46-7.37m-3.46-7.37l-3.45-7.4m-3.47-7.36l-3.45-7.38m-3.45-7.37l-3.46-7.4m-3.45-7.36l-3.45-7.38m-3.46-7.37l-3.44-7.4m-3.45-7.36l-3.44-7.38m-3.46-7.37l-3.45-7.38m-3.45-7.38l-3.46-7.37m-3.45-7.38l-3.46-7.38m-3.46-7.38l-3.45-7.37m-3.45-7.38l-3.45-7.38m-3.46-7.38l-1.72-3.68" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4791.37 676.57l-10.25-10.96 8.4-3.92zM-4928.63 383.43l10.25 10.95-8.4 3.94z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4545.44 673.74l-3.65-1.66m-7.3-3.32l-7.3-3.32m-7.28-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.33m-7.3-3.3l-7.3-3.33m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.3m-7.3-3.33l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.3l-7.3-3.33m-7.3-3.32l-7.3-3.33m-7.3-3.3l-7.3-3.34m-7.28-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.33m-7.3-3.33l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.3l-7.3-3.33m-7.3-3.32l-7.28-3.32m-7.3-3.32l-7.3-3.3m-7.3-3.33l-7.3-3.32m-7.3-3.32l-7.3-3.33m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.33l-7.28-3.33m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.3m-7.3-3.33l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.3l-7.3-3.33m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-7.3-3.32m-7.3-3.32l-3.64-1.66" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4531.54 680.06l-14.9-1.7 3.83-8.42zM-5187.05 381.84l14.9 1.7-3.84 8.43z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4884.5 666.85l-2.92-2.74m-5.84-5.46l-5.84-5.48m-5.84-5.47l-5.84-5.5m-5.84-5.46l-5.84-5.47m-5.84-5.48l-5.84-5.48m-5.84-5.48l-5.84-5.47m-5.84-5.47l-5.84-5.48m-5.84-5.47l-5.84-5.47m-5.84-5.48l-5.84-5.47m-5.84-5.48l-5.84-5.47m-5.84-5.47l-5.84-5.47m-5.84-5.47l-5.84-5.48m-5.84-5.47l-5.84-5.46m-5.84-5.48l-5.84-5.47m-5.84-5.48l-5.84-5.48m-5.84-5.47l-5.84-5.48m-5.84-5.48l-5.84-5.47m-5.84-5.48l-5.84-5.47m-5.84-5.48l-5.84-5.47m-5.84-5.47l-5.84-5.48m-5.84-5.47l-5.84-5.48m-5.84-5.47l-5.84-5.47m-5.84-5.47l-5.84-5.47m-5.84-5.47l-5.84-5.5m-5.84-5.46l-5.84-5.48m-5.84-5.47l-2.92-2.75" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4873.36 677.3l-13.58-6.4 6.34-6.75zM-5187.64 382.7l13.58 6.4-6.34 6.75z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4214.56 673.74l3.65-1.66m7.3-3.32l7.3-3.32m7.28-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.33m7.3-3.3l7.3-3.33m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.3m7.3-3.33l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.3l7.3-3.33m7.3-3.32l7.3-3.33m7.3-3.3l7.3-3.34m7.28-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.33m7.3-3.33l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.3l7.3-3.33m7.3-3.32l7.28-3.32m7.3-3.32l7.3-3.3m7.3-3.33l7.3-3.32m7.3-3.32l7.3-3.33m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.33l7.3-3.33m7.28-3.32l7.3-3.32m7.3-3.32l7.3-3.3m7.3-3.33l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.3l7.3-3.33m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l7.3-3.32m7.3-3.32l3.64-1.66" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4228.46 680.06l11.07-10.12 3.85 8.43zM-3572.95 381.84l-11.06 10.13-3.85-8.44z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4250 241V125a6 6 0 0 1 6-6h542.34a6 6 0 0 1 6 6v88.38" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4250 256.26l-4.64-14.26h9.28z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-3695.66 213.36v1.02M-4380 103.17v37.33" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-4380 87.9l4.64 14.27h-9.28zM-4380 155.76l-4.64-14.26h9.28z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-4500 46a6 6 0 0 1 6-6h228a6 6 0 0 1 6 6v32.67a6 6 0 0 1-6 6h-228a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#r" transform="matrix(1,0,0,1,-4495,45) translate(26.929012345679013 25.29861111111111)"/><use xlink:href="#s" transform="matrix(1,0,0,1,-4495,45) translate(108.54012345679013 25.29861111111111)"/><path d="M-3960.27 663.2c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm1.88-3.52c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.52c0 .56-.44 1-1 1-.55 0-1-.44-1-1 0-.55.45-1 1-1 .56 0 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.52c0 .56-.44 1-1 1-.54 0-1-.44-1-1 0-.55.46-1 1-1 .56 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm1.86-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.52c0 .55-.44 1-1 1-.54 0-1-.45-1-1s.46-1 1-1c.56 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm1.86-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.52c0 .56-.43 1-1 1-.54 0-1-.44-1-1 0-.55.46-1 1-1 .57 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm1.86-3.5c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm1.87-3.5c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.52c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.57.45-1 1-1 .56 0 1 .43 1 1zm1.88-3.52c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.87-3.5c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.57.45-1 1-1 .56 0 1 .43 1 1zm1.88-3.52c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.57.44-1 1-1 .55 0 1 .43 1 1zm1.87-3.52c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm1.87-3.52c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm1.88-3.52c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm1.88-3.52c0 .55-.44 1-1 1-.54 0-1-.45-1-1s.46-1 1-1c.56 0 1 .45 1 1zm1.88-3.5c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.57.44-1 1-1 .55 0 1 .43 1 1zm1.87-3.52c0 .55-.44 1-1 1-.54 0-1-.45-1-1s.46-1 1-1c.56 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.57 0-1-.45-1-1 0-.56.43-1 1-1 .54 0 1 .44 1 1zm1.86-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.52c0 .55-.43 1-1 1-.54 0-1-.45-1-1s.46-1 1-1c.57 0 1 .45 1 1zm1.9-3.5c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm1.86-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm1.87-3.5c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm1.87-3.5c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm1.87-3.5c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm1.88-3.52c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm1.88-3.5c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm1.87-3.5c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1z" fill="#3a414a"/><path d="M-3968.47 676.65l2.64-14.77 8.17 4.38zM-3811.53 383.35l-2.64 14.77-8.17-4.38z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-3850.03 666.3c0 .56-.45 1-1 1s-1-.44-1-1c0-.54.45-1 1-1s1 .46 1 1zm2.8-2.83c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .54-.45 1-1 1s-1-.46-1-1c0-.56.45-1 1-1s1 .44 1 1zm2.8-2.85c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .56-.44 1-1 1-.55 0-1-.44-1-1 0-.54.45-1 1-1 .56 0 1 .46 1 1zm2.8-2.83c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .54-.44 1-1 1-.54 0-1-.46-1-1 0-.56.46-1 1-1 .56 0 1 .44 1 1zm2.8-2.85c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .55 0 1 .46 1 1zm2.8-2.83c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .54-.46 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.85c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .55 0 1 .46 1 1zm2.8-2.83c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .54-.45 1-1 1-.56 0-1-.46-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.85c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .57-.45 1-1 1-.56 0-1-.43-1-1 0-.54.44-1 1-1 .55 0 1 .46 1 1zm2.8-2.83c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.85c0 .56-.46 1-1 1-.57 0-1-.44-1-1 0-.55.43-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .57-.45 1-1 1s-1-.43-1-1c0-.54.45-1 1-1s1 .46 1 1zm2.8-2.83c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.57 0-1-.45-1-1 0-.56.43-1 1-1 .54 0 1 .44 1 1zm2.78-2.85c0 .57-.45 1-1 1s-1-.43-1-1c0-.54.45-1 1-1s1 .46 1 1zm2.8-2.82c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.84c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.78-2.84c0 .55-.44 1-1 1-.55 0-1-.45-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.57 0-1-.45-1-1s.43-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.57 0-1-.44-1-1 0-.55.43-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .56-.44 1-1 1-.55 0-1-.44-1-1 0-.55.45-1 1-1 .56 0 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .54 0 1 .45 1 1zm2.78-2.84c0 .56-.45 1-1 1s-1-.44-1-1c0-.55.45-1 1-1s1 .45 1 1zm2.8-2.83c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.78-2.85c0 .56-.44 1-1 1-.55 0-1-.44-1-1 0-.55.45-1 1-1 .56 0 1 .45 1 1zm2.8-2.83c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.84c0 .54-.45 1-1 1s-1-.46-1-1c0-.57.45-1 1-1s1 .43 1 1zm2.8-2.85c0 .55-.45 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .55 0 1 .44 1 1zm2.8-2.84c0 .56-.46 1-1 1-.57 0-1-.44-1-1 0-.55.43-1 1-1 .54 0 1 .45 1 1zm2.78-2.83c0 .55-.45 1-1 1s-1-.45-1-1c0-.56.45-1 1-1s1 .44 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1zm2.78-2.84c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.85c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.57 0-1-.44-1-1 0-.54.43-1 1-1 .54 0 1 .46 1 1zm2.78-2.83c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.85c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .54 0 1 .46 1 1zm2.78-2.83c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.54 0 1 .45 1 1zm2.78-2.84c0 .54-.44 1-1 1-.55 0-1-.46-1-1 0-.56.45-1 1-1 .56 0 1 .44 1 1zm2.8-2.85c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .54 0 1 .46 1 1zm2.78-2.83c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .55-.45 1-1 1-.56 0-1-.45-1-1s.44-1 1-1c.55 0 1 .45 1 1zm2.8-2.84c0 .54-.46 1-1 1-.57 0-1-.46-1-1 0-.56.43-1 1-1 .54 0 1 .44 1 1zm2.78-2.85c0 .55-.45 1-1 1s-1-.45-1-1 .45-1 1-1 1 .45 1 1zm2.8-2.84c0 .56-.46 1-1 1-.56 0-1-.44-1-1 0-.54.44-1 1-1 .54 0 1 .46 1 1zm2.78-2.83c0 .55-.44 1-1 1-.55 0-1-.45-1-1s.45-1 1-1c.56 0 1 .45 1 1zm2.8-2.84c0 .56-.45 1-1 1-.56 0-1-.44-1-1 0-.55.44-1 1-1 .55 0 1 .45 1 1zm2.8-2.84c0 .55-.46 1-1 1-.56 0-1-.45-1-1 0-.56.44-1 1-1 .54 0 1 .44 1 1z" fill="#3a414a"/><path d="M-3861.73 677.2l6.7-13.43 6.6 6.5zM-3572.27 382.8l-6.7 13.43-6.6-6.5z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><path d="M-3658.5 775h30.26M-3658.46 775h-1.54" stroke="#3a414a" stroke-width="3" fill="none"/><path d="M-3612.47 775l-14.27 4.63v-9.26z" stroke="#3a414a" stroke-width="3" fill="#3a414a"/><path d="M-3658.5 815h7.57m15.13 0h7.56M-3658.46 815h-1.54" stroke="#3a414a" stroke-width="3" fill="none"/><path d="M-3612.47 815l-14.27 4.63v-9.26z" stroke="#3a414a" stroke-width="3" fill="#3a414a"/><path d="M-3657 848.33c0 .83-.67 1.5-1.5 1.5s-1.5-.67-1.5-1.5.67-1.5 1.5-1.5 1.5.67 1.5 1.5zm6.05 0c0 .83-.67 1.5-1.5 1.5s-1.5-.67-1.5-1.5.67-1.5 1.5-1.5 1.5.67 1.5 1.5zm6.05 0c0 .83-.67 1.5-1.5 1.5-.82 0-1.5-.67-1.5-1.5s.68-1.5 1.5-1.5c.83 0 1.5.67 1.5 1.5zm6.06 0c0 .83-.67 1.5-1.5 1.5s-1.5-.67-1.5-1.5.67-1.5 1.5-1.5 1.5.67 1.5 1.5zm6.05 0c0 .83-.66 1.5-1.5 1.5-.82 0-1.5-.67-1.5-1.5s.68-1.5 1.5-1.5c.84 0 1.5.67 1.5 1.5z" fill="#3a414a"/><path d="M-3658.46 848.33h-1.54" stroke="#3a414a" stroke-width="3" fill="none"/><path d="M-3612.47 848.33l-14.27 4.64v-9.27z" stroke="#3a414a" stroke-width="3" fill="#3a414a"/><path d="M-3607.62 766a6 6 0 0 1 6-6h92.76a6 6 0 0 1 6 6v18a6 6 0 0 1-6 6h-92.76a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#t" transform="matrix(1,0,0,1,-3602.6190476190477,765) translate(0 19.555555555555554)"/><path d="M-3607.62 839.33a6 6 0 0 1 6-6H-3446a6 6 0 0 1 6 6v18a6 6 0 0 1-6 6h-155.62a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#u" transform="matrix(1,0,0,1,-3602.6190476190477,838.3333333333334) translate(0 19.555555555555554)"/><path d="M-3607.62 806a6 6 0 0 1 6-6h113.7a6 6 0 0 1 6 6v18a6 6 0 0 1-6 6h-113.7a6 6 0 0 1-6-6z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#v" transform="matrix(1,0,0,1,-3602.6190476190473,805) translate(0 19.555555555555554)"/><path d="M-3694.76 552a12 12 0 0 1 12-12h276a12 12 0 0 1 12 12v198.96a12 12 0 0 1-12 12h-276a12 12 0 0 1-12-12z" stroke="#000" stroke-opacity="0" stroke-width="2" fill-opacity="0"/><path d="M-3638.76 613.6c0 6.63-5.37 12-12 12s-12-5.37-12-12 5.37-12 12-12 12 5.37 12 12z" stroke="#000" stroke-opacity=".25" fill="#cfe4ff"/><path d="M-3638.76 651.2c0 6.63-5.37 12-12 12s-12-5.37-12-12 5.37-12 12-12 12 5.37 12 12z" stroke="#000" stroke-opacity=".25" fill="#ffdda6"/><path d="M-3638.76 688.8c0 6.63-5.37 12-12 12s-12-5.37-12-12 5.37-12 12-12 12 5.37 12 12z" stroke="#000" stroke-opacity=".25" fill="#dedeff"/><use xlink:href="#w" transform="matrix(1,0,0,1,-3662.7619047619046,564) translate(0 19.933333333333334)"/><g><use xlink:href="#x" transform="matrix(1,0,0,1,-3626.7619047619046,601.6) translate(0 19.933333333333334)"/><use xlink:href="#y" transform="matrix(1,0,0,1,-3626.7619047619046,601.6) translate(65.57222222222221 19.933333333333334)"/></g><g><use xlink:href="#x" transform="matrix(1,0,0,1,-3626.7619047619046,639.2) translate(0 19.933333333333334)"/><use xlink:href="#z" transform="matrix(1,0,0,1,-3626.7619047619046,639.2) translate(65.57222222222221 19.933333333333334)"/></g><g><use xlink:href="#A" transform="matrix(1,0,0,1,-3626.7619047619046,676.8) translate(0 20.153333333333336)"/><use xlink:href="#B" transform="matrix(1,0,0,1,-3626.7619047619046,676.8) translate(0 52.71333333333334)"/></g><path d="M-3810 241v-20.62a6 6 0 0 1 6-6h228a6 6 0 0 1 6 6V241" stroke="#3a414a" stroke-width="2" fill="none"/><path d="M-3810 256.26l-4.64-14.26h9.28zM-3570 256.26l-4.64-14.26h9.28z" stroke="#3a414a" stroke-width="2" fill="#3a414a"/><defs><path d="M212-179c-10-28-35-45-73-45-59 0-87 40-87 99 0 60 29 101 89 101 43 0 62-24 78-52l27 14C228-24 195 4 139 4 59 4 22-46 18-125c-6-104 99-153 187-111 19 9 31 26 39 46" id="C"/><path d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="D"/><path d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="E"/><path d="M85-194c31 0 48 13 60 33l-1-100h32l1 261h-30c-2-10 0-23-3-31C134-8 116 4 85 4 32 4 16-35 15-94c0-66 23-100 70-100zm9 24c-40 0-46 34-46 75 0 40 6 74 45 74 42 0 51-32 51-76 0-42-9-74-50-73" id="F"/><path d="M24-231v-30h32v30H24zM24 0v-190h32V0H24" id="G"/><path d="M117-194c89-4 53 116 60 194h-32v-121c0-31-8-49-39-48C34-167 62-67 57 0H25l-1-190h30c1 10-1 24 2 32 11-22 29-35 61-36" id="H"/><path d="M141-36C126-15 110 5 73 4 37 3 15-17 15-53c-1-64 63-63 125-63 3-35-9-54-41-54-24 1-41 7-42 31l-33-3c5-37 33-52 76-52 45 0 72 20 72 64v82c-1 20 7 32 28 27v20c-31 9-61-2-59-35zM48-53c0 20 12 33 32 33 41-3 63-29 60-74-43 2-92-5-92 41" id="I"/><path d="M59-47c-2 24 18 29 38 22v24C64 9 27 4 27-40v-127H5v-23h24l9-43h21v43h35v23H59v120" id="J"/><g id="a"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#C"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.83333333333333,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,52.8827160493827,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,62.43209876543209,0)" xlink:href="#F"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,78.48148148148147,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.82098765432096,0)" xlink:href="#H"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,100.87037037037035,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,116.91975308641973,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,124.94444444444444,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,140.99382716049382,0)" xlink:href="#E"/></g><path d="M140-251c81 0 123 46 123 126C263-46 219 4 140 4 59 4 17-45 17-125s42-126 123-126zm0 227c63 0 89-41 89-101s-29-99-89-99c-61 0-89 39-89 99S79-25 140-24" id="K"/><path d="M108 0H70L1-190h34L89-25l56-165h34" id="L"/><path d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="M"/><path d="M24 0v-261h32V0H24" id="N"/><g id="b"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#K"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,22.469135802469136,0)" xlink:href="#L"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.91358024691358,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,52.96296296296296,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,62.51234567901234,0)" xlink:href="#N"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,68.85185185185185,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.90123456790123,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,94.4506172839506,0)" xlink:href="#F"/></g><path d="M240 0l2-218c-23 76-54 145-80 218h-23L58-218 59 0H30v-248h44l77 211c21-75 51-140 76-211h43V0h-30" id="O"/><path d="M177-190C167-65 218 103 67 71c-23-6-38-20-44-43l32-5c15 47 100 32 89-28v-30C133-14 115 1 83 1 29 1 15-40 15-95c0-56 16-97 71-98 29-1 48 16 59 35 1-10 0-23 2-32h30zM94-22c36 0 50-32 50-73 0-42-14-75-50-75-39 0-46 34-46 75s6 73 46 73" id="P"/><g id="c"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#O"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,23.993827160493826,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,30.333333333333332,0)" xlink:href="#F"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,46.382716049382715,0)" xlink:href="#F"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,62.4320987654321,0)" xlink:href="#N"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,68.77160493827161,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.82098765432099,0)" xlink:href="#O"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,108.81481481481481,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,124.86419753086419,0)" xlink:href="#H"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,140.91358024691357,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,156.96296296296296,0)" xlink:href="#P"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,173.01234567901238,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,189.06172839506175,0)" xlink:href="#E"/></g><path d="M160-131c35 5 61 23 61 61C221 17 115-2 30 0v-248c76 3 177-17 177 60 0 33-19 50-47 57zm-97-11c50-1 110 9 110-42 0-47-63-36-110-37v79zm0 115c55-2 124 14 124-45 0-56-70-42-124-44v89" id="Q"/><path d="M143 0L79-87 56-68V0H24v-261h32v163l83-92h37l-77 82L181 0h-38" id="R"/><g id="d"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#Q"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,19.25925925925926,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,28.80864197530864,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,44.858024691358025,0)" xlink:href="#R"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,59.30246913580247,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,75.35185185185185,0)" xlink:href="#E"/></g><path d="M233-177c-1 41-23 64-60 70L243 0h-38l-65-103H63V0H30v-248c88 3 205-21 203 71zM63-129c60-2 137 13 137-47 0-61-80-42-137-45v92" id="S"/><path d="M84 4C-5 8 30-112 23-190h32v120c0 31 7 50 39 49 72-2 45-101 50-169h31l1 190h-30c-1-10 1-25-2-33-11 22-28 36-60 37" id="T"/><g id="e"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#S"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.83333333333333,0)" xlink:href="#T"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,52.8827160493827,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,60.9074074074074,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,76.95679012345678,0)" xlink:href="#E"/></g><path d="M197 0v-115H63V0H30v-248h33v105h134v-105h34V0h-34" id="U"/><path d="M135-143c-3-34-86-38-87 0 15 53 115 12 119 90S17 21 10-45l28-5c4 36 97 45 98 0-10-56-113-15-118-90-4-57 82-63 122-42 12 7 21 19 24 35" id="V"/><path d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="W"/><g id="f"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#U"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,27.123456790123456,0)" xlink:href="#V"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,41.5679012345679,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,49.59259259259259,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,65.64197530864197,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,75.19135802469135,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,81.53086419753086,0)" xlink:href="#W"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,95.9753086419753,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,112.02469135802468,0)" xlink:href="#N"/></g><path d="M230 0l2-204L168 0h-37L68-204 70 0H24v-248h70l56 185 57-185h69V0h-46" id="X"/><path d="M133-34C117-15 103 5 69 4 32 3 11-16 11-54c-1-60 55-63 116-61 1-26-3-47-28-47-18 1-26 9-28 27l-52-2c7-38 36-58 82-57s74 22 75 68l1 82c-1 14 12 18 25 15v27c-30 8-71 5-69-32zm-48 3c29 0 43-24 42-57-32 0-66-3-65 30 0 17 8 27 23 27" id="Y"/><path d="M137-138c1-29-70-34-71-4 15 46 118 7 119 86 1 83-164 76-172 9l43-7c4 19 20 25 44 25 33 8 57-30 24-41C81-84 22-81 20-136c-2-80 154-74 161-7" id="Z"/><path d="M115-3C79 11 28 4 28-45v-112H4v-33h27l15-45h31v45h36v33H77v99c-1 23 16 31 38 25v30" id="aa"/><path d="M185-48c-13 30-37 53-82 52C43 2 14-33 14-96s30-98 90-98c62 0 83 45 84 108H66c0 31 8 55 39 56 18 0 30-7 34-22zm-45-69c5-46-57-63-70-21-2 6-4 13-4 21h74" id="ab"/><path d="M135-150c-39-12-60 13-60 57V0H25l-1-190h47c2 13-1 29 3 40 6-28 27-53 61-41v41" id="ac"/><g id="g"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#X"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,23.993827160493826,0)" xlink:href="#Y"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,40.04320987654321,0)" xlink:href="#Z"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,56.092592592592595,0)" xlink:href="#aa"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,65.64197530864197,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,81.69135802469135,0)" xlink:href="#ac"/></g><path d="M128 0H69L1-190h53L99-40l48-150h52" id="ad"/><g id="h"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#Z"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,16.049382716049383,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,32.098765432098766,0)" xlink:href="#ac"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,43.33333333333334,0)" xlink:href="#ad"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,59.38271604938272,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,75.4320987654321,0)" xlink:href="#ac"/></g><path d="M140-251c80 0 125 45 125 126 0 70-33 111-92 124 6 28 34 38 68 31l-1 36C177 83 129 55 120 2 51-6 15-50 15-125c0-81 44-126 125-126zm-1 214c52 0 73-35 73-88 0-50-21-86-72-86-52 0-73 35-73 86s22 88 72 88" id="ae"/><path d="M85 4C-2 5 27-109 22-190h50c7 57-23 150 33 157 60-5 35-97 40-157h50l1 190h-47c-2-12 1-28-3-38-12 25-28 42-61 42" id="af"/><path d="M123 10C108 53 80 86 19 72V37c35 8 53-11 59-39L3-190h52l48 148c12-52 28-100 44-148h51" id="ag"/><g id="i"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#ae"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,22.469135802469136,0)" xlink:href="#af"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,40.04320987654321,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,56.092592592592595,0)" xlink:href="#ac"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,67.32716049382717,0)" xlink:href="#ag"/></g><path d="M24-248c120-7 223 5 221 122C244-46 201 0 124 0H24v-248zM76-40c74 7 117-18 117-86 0-67-45-88-117-82v168" id="ah"/><g id="j"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#ah"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#Y"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.83333333333333,0)" xlink:href="#aa"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,46.3827160493827,0)" xlink:href="#Y"/></g><path fill="#3a414a" d="M30-248c87 1 191-15 191 75 0 78-77 80-158 76V0H30v-248zm33 125c57 0 124 11 124-50 0-59-68-47-124-48v98" id="ai"/><path fill="#3a414a" d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="aj"/><path fill="#3a414a" d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="ak"/><path fill="#3a414a" d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="al"/><path fill="#3a414a" d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="am"/><path fill="#3a414a" d="M135-143c-3-34-86-38-87 0 15 53 115 12 119 90S17 21 10-45l28-5c4 36 97 45 98 0-10-56-113-15-118-90-4-57 82-63 122-42 12 7 21 19 24 35" id="an"/><g id="k"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#ai"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,14.814814814814813,0)" xlink:href="#aj"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,22.160493827160494,0)" xlink:href="#ak"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,34.50617283950617,0)" xlink:href="#al"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,45.61728395061728,0)" xlink:href="#am"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,57.962962962962955,0)" xlink:href="#an"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,69.07407407407406,0)" xlink:href="#an"/></g><path d="M209 0H11v-25l151-195H24v-28h176v25L50-27h159V0" id="ao"/><path d="M194 0L95-120 63-95V0H30v-248h33v124l119-124h40L117-140 236 0h-42" id="ap"/><path d="M115-194c55 1 70 41 70 98S169 2 115 4C84 4 66-9 55-30l1 105H24l-1-265h31l2 30c10-21 28-34 59-34zm-8 174c40 0 45-34 45-75s-6-73-45-74c-42 0-51 32-51 76 0 43 10 73 51 73" id="aq"/><g id="l"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#ao"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,17.574074074074073,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,33.623456790123456,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,49.672839506172835,0)" xlink:href="#ap"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,68.9320987654321,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.98148148148148,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,101.03086419753086,0)" xlink:href="#aq"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,117.08024691358024,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,133.12962962962962,0)" xlink:href="#E"/></g><path d="M30-248c118-7 216 8 213 122C240-48 200 0 122 0H30v-248zM63-27c89 8 146-16 146-99s-60-101-146-95v194" id="ar"/><g id="m"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#ar"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,36.83333333333333,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,52.8827160493827,0)" xlink:href="#aq"/></g><g id="n"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#V"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,14.444444444444443,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,22.469135802469136,0)" xlink:href="#D"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,38.51851851851852,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,48.0679012345679,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,64.11728395061728,0)" xlink:href="#P"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,80.16666666666666,0)" xlink:href="#M"/></g><g id="o"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#O"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,23.993827160493826,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,40.04320987654321,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,48.0679012345679,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,64.11728395061728,0)" xlink:href="#F"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,80.16666666666666,0)" xlink:href="#I"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,96.21604938271605,0)" xlink:href="#J"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,104.24074074074073,0)" xlink:href="#I"/></g><path d="M24 0v-248h195v40H76v63h132v40H76v65h150V0H24" id="as"/><path d="M144 0l-44-69L55 0H2l70-98-66-92h53l41 62 40-62h54l-67 91 71 99h-54" id="at"/><path d="M135-194c87-1 58 113 63 194h-50c-7-57 23-157-34-157-59 0-34 97-39 157H25l-1-190h47c2 12-1 28 3 38 12-26 28-41 61-42" id="au"/><path d="M25 0v-261h50V0H25" id="av"/><g id="p"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#as"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,19.25925925925926,0)" xlink:href="#at"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,35.308641975308646,0)" xlink:href="#aa"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,44.858024691358025,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,60.907407407407405,0)" xlink:href="#ac"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,72.14197530864197,0)" xlink:href="#au"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,89.71604938271605,0)" xlink:href="#Y"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,105.76543209876543,0)" xlink:href="#av"/></g><path d="M88-194c31-1 46 15 58 34l-1-101h50l1 261h-48c-2-10 0-23-3-31C134-8 116 4 84 4 32 4 16-41 15-95c0-56 19-97 73-99zm17 164c33 0 40-30 41-66 1-37-9-64-41-64s-38 30-39 65c0 43 13 65 39 65" id="aw"/><path d="M135-194c53 0 70 44 70 98 0 56-19 98-73 100-31 1-45-17-59-34 3 33 2 69 2 105H25l-1-265h48c2 10 0 23 3 31 11-24 29-35 60-35zM114-30c33 0 39-31 40-66 0-38-9-64-40-64-56 0-55 130 0 130" id="ax"/><path d="M190-63c-7 42-38 67-86 67-59 0-84-38-90-98-12-110 154-137 174-36l-49 2c-2-19-15-32-35-32-30 0-35 28-38 64-6 74 65 87 74 30" id="ay"/><path d="M25-224v-37h50v37H25zM25 0v-190h50V0H25" id="az"/><g id="q"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#aw"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,17.574074074074073,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,33.623456790123456,0)" xlink:href="#ax"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,51.197530864197525,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,67.24691358024691,0)" xlink:href="#au"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,84.82098765432099,0)" xlink:href="#aw"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,102.39506172839505,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,118.44444444444444,0)" xlink:href="#au"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,136.0185185185185,0)" xlink:href="#ay"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,152.0679012345679,0)" xlink:href="#az"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,160.09259259259258,0)" xlink:href="#ab"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,176.14197530864197,0)" xlink:href="#Z"/></g><g id="r"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#C"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,20.78395061728395,0)" xlink:href="#N"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,27.123456790123456,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,33.46296296296296,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,49.51234567901234,0)" xlink:href="#H"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,65.56172839506172,0)" xlink:href="#J"/></g><path d="M145-31C134-9 116 4 85 4 32 4 16-35 15-94c0-59 17-99 70-100 32-1 48 14 60 33 0-11-1-24 2-32h30l-1 268h-32zM93-21c41 0 51-33 51-76s-8-73-50-73c-40 0-46 35-46 75s5 74 45 74" id="aA"/><g id="s"><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,0,0)" xlink:href="#aA"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,16.049382716049383,0)" xlink:href="#T"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,32.098765432098766,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,48.14814814814815,0)" xlink:href="#E"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,57.697530864197525,0)" xlink:href="#G"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,64.03703703703704,0)" xlink:href="#M"/><use transform="matrix(0.08024691358024691,0,0,0.08024691358024691,80.08641975308642,0)" xlink:href="#V"/></g><path d="M140-251c81 0 123 46 123 126C263-53 228-8 163 1c7 30 30 48 69 40v23c-55 16-95-15-103-61C56-3 17-48 17-125c0-80 42-126 123-126zm0 227c63 0 89-41 89-101s-29-99-89-99c-61 0-89 39-89 99S79-25 140-24" id="aB"/><g id="t"><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,0,0)" xlink:href="#aB"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,19.01234567901234,0)" xlink:href="#T"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,32.59259259259258,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,46.17283950617282,0)" xlink:href="#E"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,54.25308641975307,0)" xlink:href="#G"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,59.617283950617264,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,73.1975308641975,0)" xlink:href="#V"/></g><path d="M0 4l72-265h28L28 4H0" id="aC"/><path d="M185-189c-5-48-123-54-124 2 14 75 158 14 163 119 3 78-121 87-175 55-17-10-28-26-33-46l33-7c5 56 141 63 141-1 0-78-155-14-162-118-5-82 145-84 179-34 5 7 8 16 11 25" id="aD"/><path d="M210-169c-67 3-38 105-44 169h-31v-121c0-29-5-50-35-48C34-165 62-65 56 0H25l-1-190h30c1 10-1 24 2 32 10-44 99-50 107 0 11-21 27-35 58-36 85-2 47 119 55 194h-31v-121c0-29-5-49-35-48" id="aE"/><g id="u"><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,0,0)" xlink:href="#ar"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,17.586419753086414,0)" xlink:href="#I"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,31.166666666666657,0)" xlink:href="#J"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,37.95679012345678,0)" xlink:href="#I"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,51.537037037037024,0)" xlink:href="#aC"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,58.327160493827144,0)" xlink:href="#aD"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,74.62345679012344,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,88.20370370370368,0)" xlink:href="#P"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,101.78395061728392,0)" xlink:href="#aE"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,122.08641975308639,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,135.66666666666663,0)" xlink:href="#H"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,149.24691358024688,0)" xlink:href="#J"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,156.037037037037,0)" xlink:href="#V"/></g><g id="v"><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,0,0)" xlink:href="#O"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,20.302469135802465,0)" xlink:href="#M"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,33.88271604938271,0)" xlink:href="#J"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,40.67283950617283,0)" xlink:href="#I"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,54.253086419753075,0)" xlink:href="#F"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,67.83333333333331,0)" xlink:href="#I"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,81.41358024691355,0)" xlink:href="#J"/><use transform="matrix(0.06790123456790122,0,0,0.06790123456790122,88.20370370370368,0)" xlink:href="#I"/></g><g id="x"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#ar"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,17.746296296296293,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,25.899999999999995,0)" xlink:href="#T"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,39.603703703703694,0)" xlink:href="#G"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,45.01666666666666,0)" xlink:href="#F"/></g><g id="y"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#V"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,12.33333333333333,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,26.03703703703703,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,34.190740740740736,0)" xlink:href="#L"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,46.524074074074065,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,60.22777777777777,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,68.38148148148147,0)" xlink:href="#V"/></g><g id="z"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#V"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,12.33333333333333,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,26.03703703703703,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,34.190740740740736,0)" xlink:href="#L"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,46.524074074074065,0)" xlink:href="#G"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,51.93703703703703,0)" xlink:href="#W"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,64.27037037037036,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,77.97407407407405,0)" xlink:href="#V"/></g><path d="M30 0v-248h187v28H63v79h144v27H63v87h162V0H30" id="aF"/><path d="M141 0L90-78 38 0H4l68-98-65-92h35l48 74 47-74h35l-64 92 68 98h-35" id="aG"/><g id="A"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#aF"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,16.444444444444443,0)" xlink:href="#aG"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,28.77777777777777,0)" xlink:href="#J"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,35.62962962962961,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,49.33333333333332,0)" xlink:href="#E"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,57.48703703703703,0)" xlink:href="#H"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,71.19074074074072,0)" xlink:href="#I"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,84.89444444444442,0)" xlink:href="#N"/></g><g id="B"><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,0,0)" xlink:href="#F"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,13.7037037037037,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,27.4074074074074,0)" xlink:href="#aq"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,41.11111111111111,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,54.8148148148148,0)" xlink:href="#H"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,68.5185185185185,0)" xlink:href="#F"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,82.22222222222221,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,95.9259259259259,0)" xlink:href="#H"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,109.62962962962959,0)" xlink:href="#G"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,115.04259259259256,0)" xlink:href="#M"/><use transform="matrix(0.0685185185185185,0,0,0.0685185185185185,128.74629629629626,0)" xlink:href="#V"/></g></defs></g></svg>
\ No newline at end of file

From bfbd21bce01f600b7d899c1360280466435c8bb1 Mon Sep 17 00:00:00 2001
From: AmatyaAvadhanula <amatya.avadhanula@imply.io>
Date: Wed, 4 Sep 2024 23:36:49 +0530
Subject: [PATCH 87/99] Revert "Add integration tests for concurrent append and
 replace (#16755)" (#17000)

This reverts commit 70bad948e379dc07911d896bfcd23b5a8a149e32.
---
 .../clients/OverlordResourceTestClient.java   |  26 -
 .../duty/ITConcurrentAppendReplaceTest.java   | 458 ------------------
 .../indexer/AbstractStreamIndexingTest.java   |   3 -
 ...r_with_concurrent_locks_spec_template.json |  60 ---
 4 files changed, 547 deletions(-)
 delete mode 100644 integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITConcurrentAppendReplaceTest.java
 delete mode 100644 integration-tests/src/test/resources/stream/data/supervisor_with_concurrent_locks_spec_template.json

diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java
index 5a821691b2e..f75dc6043f9 100644
--- a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java
+++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java
@@ -28,7 +28,6 @@ import org.apache.druid.indexer.TaskState;
 import org.apache.druid.indexer.TaskStatusPlus;
 import org.apache.druid.indexer.report.IngestionStatsAndErrorsTaskReport;
 import org.apache.druid.indexer.report.TaskReport;
-import org.apache.druid.indexing.overlord.http.TaskLockResponse;
 import org.apache.druid.indexing.overlord.http.TaskPayloadResponse;
 import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager;
 import org.apache.druid.java.util.common.ISE;
@@ -361,31 +360,6 @@ public class OverlordResourceTestClient
     }
   }
 
-  public TaskLockResponse getActiveLocks(List<LockFilterPolicy> lockFilterPolicies)
-  {
-    try {
-      String jsonBody = jsonMapper.writeValueAsString(lockFilterPolicies);
-
-      StatusResponseHolder response = httpClient.go(
-          new Request(HttpMethod.POST, new URL(getIndexerURL() + "activeLocks"))
-              .setContent(
-                  "application/json",
-                  StringUtils.toUtf8(jsonBody)
-              ),
-          StatusResponseHandler.getInstance()
-      ).get();
-      return jsonMapper.readValue(
-          response.getContent(),
-          new TypeReference<TaskLockResponse>()
-          {
-          }
-      );
-    }
-    catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
   public void waitUntilTaskCompletes(final String taskID)
   {
     waitUntilTaskCompletes(taskID, ITRetryUtil.DEFAULT_RETRY_SLEEP, ITRetryUtil.DEFAULT_RETRY_COUNT);
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITConcurrentAppendReplaceTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITConcurrentAppendReplaceTest.java
deleted file mode 100644
index 94633d12366..00000000000
--- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITConcurrentAppendReplaceTest.java
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.tests.coordinator.duty;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.inject.Inject;
-import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
-import org.apache.druid.indexer.partitions.PartitionsSpec;
-import org.apache.druid.indexing.common.TaskLock;
-import org.apache.druid.indexing.common.TaskLockType;
-import org.apache.druid.indexing.common.task.Tasks;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.metadata.LockFilterPolicy;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
-import org.apache.druid.server.coordinator.DruidCompactionConfig;
-import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig;
-import org.apache.druid.testing.clients.CompactionResourceTestClient;
-import org.apache.druid.testing.clients.TaskResponseObject;
-import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.testing.utils.EventSerializer;
-import org.apache.druid.testing.utils.ITRetryUtil;
-import org.apache.druid.testing.utils.KafkaUtil;
-import org.apache.druid.testing.utils.MsqTestQueryHelper;
-import org.apache.druid.testing.utils.StreamEventWriter;
-import org.apache.druid.testing.utils.StreamGenerator;
-import org.apache.druid.testing.utils.WikipediaStreamEventStreamGenerator;
-import org.apache.druid.tests.TestNGGroup;
-import org.apache.druid.tests.indexer.AbstractKafkaIndexingServiceTest;
-import org.apache.druid.tests.indexer.AbstractStreamIndexingTest;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.utils.CollectionUtils;
-import org.joda.time.Interval;
-import org.joda.time.Period;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import java.io.Closeable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Function;
-
-/**
- * Integration Test to verify behaviour when there are concurrent
- * compaction tasks with ongoing stream ingestion tasks.
- */
-@Test(groups = {TestNGGroup.COMPACTION})
-@Guice(moduleFactory = DruidTestModuleFactory.class)
-public class ITConcurrentAppendReplaceTest extends AbstractKafkaIndexingServiceTest
-{
-  private static final Logger LOG = new Logger(ITConcurrentAppendReplaceTest.class);
-
-  @Inject
-  private CompactionResourceTestClient compactionResource;
-  
-  @Inject
-  private MsqTestQueryHelper msqHelper;
-
-  private GeneratedTestConfig generatedTestConfig;
-  private StreamGenerator streamGenerator;
-
-  private String fullDatasourceName;
-
-  private int currentRowCount = 0;
-  private boolean concurrentAppendAndReplaceLocksExisted;
-
-  @DataProvider
-  public static Object[] getKafkaTransactionStatesForTest()
-  {
-    return new Object[]{false, true};
-  }
-
-  @BeforeClass
-  public void setupClass() throws Exception
-  {
-    doBeforeClass();
-  }
-
-  @BeforeMethod
-  public void setup() throws Exception
-  {
-    generatedTestConfig = new GeneratedTestConfig(
-        Specs.PARSER_TYPE,
-        getResourceAsString(Specs.INPUT_FORMAT_PATH)
-    );
-    fullDatasourceName = generatedTestConfig.getFullDatasourceName();
-    final EventSerializer serializer = jsonMapper.readValue(
-        getResourceAsStream(Specs.SERIALIZER_PATH),
-        EventSerializer.class
-    );
-    streamGenerator = new WikipediaStreamEventStreamGenerator(serializer, 6, 100);
-    concurrentAppendAndReplaceLocksExisted = false;
-  }
-
-  @Override
-  public String getTestNamePrefix()
-  {
-    return "autocompact_lock_contention";
-  }
-
-  @Test(dataProvider = "getKafkaTransactionStatesForTest")
-  public void testConcurrentStreamAppendReplace(boolean transactionEnabled) throws Exception
-  {
-    if (shouldSkipTest(transactionEnabled)) {
-      return;
-    }
-
-    try (
-        final Closeable closer = createResourceCloser(generatedTestConfig);
-        final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled)
-    ) {
-      // Start supervisor
-      final String taskSpec
-          = generatedTestConfig.getStreamIngestionPropsTransform()
-                               .apply(getResourceAsString(SUPERVISOR_WITH_CONCURRENT_LOCKS_SPEC_TEMPLATE_PATH));
-      generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec));
-      LOG.info("supervisorSpec: [%s]", taskSpec);
-
-      // Generate data for minutes 1, 2 and 3
-      final Interval minute1 = Intervals.of("2000-01-01T01:01:00Z/2000-01-01T01:02:00Z");
-      long rowsForMinute1 = generateData(minute1, streamEventWriter);
-
-      final Interval minute2 = Intervals.of("2000-01-01T01:02:00Z/2000-01-01T01:03:00Z");
-      long rowsForMinute2 = generateData(minute2, streamEventWriter);
-
-      final Interval minute3 = Intervals.of("2000-01-01T01:03:00Z/2000-01-01T01:04:00Z");
-      long rowsForMinute3 = generateData(minute3, streamEventWriter);
-
-      Function<String, AggregatorFactory> function = name -> new LongSumAggregatorFactory(name, "count");
-
-      // Wait for data to be ingested for all the minutes
-      ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
-
-      ensureSegmentsLoaded(fullDatasourceName);
-
-      // 2 segments for each minute, total 6
-      ensureSegmentsCount(fullDatasourceName, 6);
-
-      // Trigger auto compaction
-      submitAndVerifyCompactionConfig(fullDatasourceName, null);
-      compactionResource.forceTriggerAutoCompaction();
-
-      // Verify that all the segments are now compacted
-      ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
-      ensureSegmentsLoaded(fullDatasourceName);
-      ensureSegmentsCount(fullDatasourceName, 3);
-      verifyCompactedIntervals(fullDatasourceName, minute1, minute2, minute3);
-
-      // Concurrent compaction with configured segment granularity
-      for (int i = 0; i < 5; i++) {
-        rowsForMinute1 += generateData(minute1, streamEventWriter);
-        rowsForMinute2 += generateData(minute2, streamEventWriter);
-        rowsForMinute3 += generateData(minute3, streamEventWriter);
-
-        compactionResource.forceTriggerAutoCompaction();
-        ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
-        checkAndSetConcurrentLocks(fullDatasourceName);
-      }
-
-      // Verify the state with minute granularity
-      ensureSegmentsCount(fullDatasourceName, 3);
-      ensureSegmentsLoaded(fullDatasourceName);
-      ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
-      verifyCompactedIntervals(fullDatasourceName, minute1, minute2, minute3);
-
-
-      // Use ALL segment granularity for compaction and run concurrent streaming ingestion
-      submitAndVerifyCompactionConfig(fullDatasourceName, Granularities.DAY);
-
-      for (int i = 0; i < 5; i++) {
-        rowsForMinute1 += generateData(minute1, streamEventWriter);
-        rowsForMinute2 += generateData(minute2, streamEventWriter);
-        rowsForMinute3 += generateData(minute3, streamEventWriter);
-
-        compactionResource.forceTriggerAutoCompaction();
-        ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
-        checkAndSetConcurrentLocks(fullDatasourceName);
-      }
-
-      // Verify the state with all granularity
-      ensureSegmentsCount(fullDatasourceName, 1);
-      ensureSegmentsLoaded(fullDatasourceName);
-      ensureRowCount(fullDatasourceName, rowsForMinute1 + rowsForMinute2 + rowsForMinute3, function);
-      verifyCompactedIntervals(fullDatasourceName, Intervals.of("2000-01-01/2000-01-02"));
-
-      Assert.assertTrue(concurrentAppendAndReplaceLocksExisted);
-    }
-  }
-
-    /**
-     * Retries until the segment count is as expected.
-     */
-  private void ensureSegmentsCount(String datasource, int numExpectedSegments)
-  {
-    ITRetryUtil.retryUntilTrue(
-        () -> {
-          compactionResource.forceTriggerAutoCompaction();
-          printTaskStatuses(datasource);
-          checkAndSetConcurrentLocks(datasource);
-          List<DataSegment> segments = coordinator.getFullSegmentsMetadata(datasource);
-          StringBuilder sb = new StringBuilder();
-          segments.forEach(
-              seg -> sb.append("{")
-                       .append(seg.getId())
-                       .append(", ")
-                       .append(seg.getSize())
-                       .append("}, ")
-          );
-          LOG.info("Found Segments: %s", sb);
-          LOG.info("Current metadata segment count: %d, expected: %d", segments.size(), numExpectedSegments);
-          return segments.size() == numExpectedSegments;
-        },
-        "Segment count check"
-    );
-  }
-
-  /**
-   * Verifies that the given intervals have been compacted.
-   */
-  private void verifyCompactedIntervals(String datasource, Interval... compactedIntervals)
-  {
-    List<DataSegment> segments = coordinator.getFullSegmentsMetadata(datasource);
-    List<DataSegment> observedCompactedSegments = new ArrayList<>();
-    Set<Interval> observedCompactedIntervals = new HashSet<>();
-    for (DataSegment segment : segments) {
-      if (segment.getLastCompactionState() != null) {
-        observedCompactedSegments.add(segment);
-        observedCompactedIntervals.add(segment.getInterval());
-      }
-    }
-
-    Set<Interval> expectedCompactedIntervals = new HashSet<>(Arrays.asList(compactedIntervals));
-    Assert.assertEquals(observedCompactedIntervals, expectedCompactedIntervals);
-
-    DynamicPartitionsSpec expectedPartitionSpec = new DynamicPartitionsSpec(
-        PartitionsSpec.DEFAULT_MAX_ROWS_PER_SEGMENT,
-        Long.MAX_VALUE
-    );
-    for (DataSegment compactedSegment : observedCompactedSegments) {
-      Assert.assertNotNull(compactedSegment.getLastCompactionState());
-      Assert.assertEquals(
-          compactedSegment.getLastCompactionState().getPartitionsSpec(),
-          expectedPartitionSpec
-      );
-    }
-  }
-
-  /**
-   * Generates data points for the specified interval.
-   *
-   * @return Number of rows generated.
-   */
-  private long generateData(Interval interval, StreamEventWriter streamEventWriter)
-  {
-    long rowCount = streamGenerator.run(
-        generatedTestConfig.getStreamName(),
-        streamEventWriter,
-        20,
-        interval.getStart()
-    );
-    LOG.info("Generated %d Rows for Interval [%s]", rowCount, interval);
-
-    return rowCount;
-  }
-
-  /**
-   * Retries until segments have been loaded.
-   */
-  private void ensureSegmentsLoaded(String datasource)
-  {
-    ITRetryUtil.retryUntilTrue(
-        () -> coordinator.areSegmentsLoaded(datasource),
-        "Segment Loading"
-    );
-  }
-
-  private void checkAndSetConcurrentLocks(String datasource)
-  {
-    LockFilterPolicy lockFilterPolicy = new LockFilterPolicy(datasource, 0, null, null);
-    final List<TaskLock> locks = indexer.getActiveLocks(ImmutableList.of(lockFilterPolicy))
-                                        .getDatasourceToLocks()
-                                        .get(datasource);
-    if (CollectionUtils.isNullOrEmpty(locks)) {
-      return;
-    }
-    Set<Interval> replaceIntervals = new HashSet<>();
-    Set<Interval> appendIntervals = new HashSet<>();
-    for (TaskLock lock : locks) {
-      if (lock.getType() == TaskLockType.APPEND) {
-        appendIntervals.add(lock.getInterval());
-      } else if (lock.getType() == TaskLockType.REPLACE) {
-        replaceIntervals.add(lock.getInterval());
-      }
-    }
-
-    for (Interval appendInterval : appendIntervals) {
-      for (Interval replaceInterval : replaceIntervals) {
-        if (appendInterval.overlaps(replaceInterval)) {
-          concurrentAppendAndReplaceLocksExisted = true;
-          return;
-        }
-      }
-    }
-  }
-
-  /**
-   * Checks if a test should be skipped based on whether transaction is enabled or not.
-   */
-  private boolean shouldSkipTest(boolean testEnableTransaction)
-  {
-    Map<String, String> kafkaTestProps = KafkaUtil
-        .getAdditionalKafkaTestConfigFromProperties(config);
-    boolean configEnableTransaction = Boolean.parseBoolean(
-        kafkaTestProps.getOrDefault(KafkaUtil.TEST_CONFIG_TRANSACTION_ENABLED, "false")
-    );
-
-    return configEnableTransaction != testEnableTransaction;
-  }
-
-  /**
-   * Submits a compaction config for the current datasource.
-   */
-  private void submitAndVerifyCompactionConfig(String datasource, Granularity segmentGranularity) throws Exception
-  {
-    final UserCompactionTaskGranularityConfig granularitySpec =
-        new UserCompactionTaskGranularityConfig(segmentGranularity, null, null);
-    final DataSourceCompactionConfig compactionConfig =
-        DataSourceCompactionConfig.builder()
-                                  .forDataSource(datasource)
-                                  .withSkipOffsetFromLatest(Period.ZERO)
-                                  .withGranularitySpec(granularitySpec)
-                                  .withTaskContext(ImmutableMap.of(Tasks.USE_CONCURRENT_LOCKS, true))
-                                  .build();
-    compactionResource.updateCompactionTaskSlot(0.5, 10, null);
-    compactionResource.submitCompactionConfig(compactionConfig);
-
-    // Wait for compaction config to persist
-    Thread.sleep(2000);
-
-    // Verify that the compaction config is updated correctly.
-    DruidCompactionConfig druidCompactionConfig = compactionResource.getCompactionConfig();
-    DataSourceCompactionConfig observedCompactionConfig = null;
-    for (DataSourceCompactionConfig dataSourceCompactionConfig : druidCompactionConfig.getCompactionConfigs()) {
-      if (dataSourceCompactionConfig.getDataSource().equals(datasource)) {
-        observedCompactionConfig = dataSourceCompactionConfig;
-      }
-    }
-    Assert.assertEquals(observedCompactionConfig, compactionConfig);
-
-    observedCompactionConfig = compactionResource.getDataSourceCompactionConfig(datasource);
-    Assert.assertEquals(observedCompactionConfig, compactionConfig);
-  }
-
-  /**
-   * Gets the number of complete compaction tasks.
-   */
-  private void printTaskStatuses(String datsource)
-  {
-    List<TaskResponseObject> incompleteTasks = indexer
-        .getUncompletedTasksForDataSource(datsource);
-    List<TaskResponseObject> completeTasks = indexer
-        .getCompleteTasksForDataSource(datsource);
-
-    printTasks(incompleteTasks, "Incomplete");
-    printTasks(completeTasks, "Complete");
-  }
-
-  private void printTasks(List<TaskResponseObject> tasks, String taskState)
-  {
-    StringBuilder sb = new StringBuilder();
-    tasks.forEach(
-        task -> sb.append("{")
-                  .append(task.getType())
-                  .append(", ")
-                  .append(task.getStatus())
-                  .append(", ")
-                  .append(task.getCreatedTime())
-                  .append("}, ")
-    );
-    LOG.info("%s Tasks: %s", taskState, sb);
-  }
-
-  /**
-   * Retries until the total row count is as expected.
-   * Also verifies that row count is non-decreasing
-   */
-  private void ensureRowCount(String datasource, long totalRows, Function<String, AggregatorFactory> function)
-  {
-    LOG.info("Verifying Row Count. Expected: %s", totalRows);
-    ITRetryUtil.retryUntilTrue(
-        () -> {
-          compactionResource.forceTriggerAutoCompaction();
-          printTaskStatuses(datasource);
-          checkAndSetConcurrentLocks(datasource);
-          int newRowCount = this.queryHelper.countRows(
-              datasource,
-              Intervals.ETERNITY,
-              function
-          );
-          Assert.assertTrue(newRowCount >= currentRowCount, "Number of events queried must be non decreasing");
-          currentRowCount = newRowCount;
-          return currentRowCount == totalRows;
-        },
-        StringUtils.format(
-            "dataSource[%s] has [%,d] queryable rows, expected [%,d]",
-            datasource,
-            this.queryHelper.countRows(
-                datasource,
-                Intervals.ETERNITY,
-                function
-            ),
-            totalRows
-        )
-    );
-  }
-
-  /**
-   * Constants for test specs.
-   */
-  private static class Specs
-  {
-    static final String SERIALIZER_PATH = DATA_RESOURCE_ROOT + "/csv/serializer/serializer.json";
-    static final String INPUT_FORMAT_PATH = DATA_RESOURCE_ROOT + "/csv/input_format/input_format.json";
-    static final String PARSER_TYPE = AbstractStreamIndexingTest.INPUT_FORMAT;
-  }
-
-}
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
index f09eecea5b5..8cc8388ba47 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java
@@ -80,7 +80,6 @@ public abstract class AbstractStreamIndexingTest extends AbstractIndexerTest
 
   private static final String QUERIES_FILE = "/stream/queries/stream_index_queries.json";
   private static final String SUPERVISOR_SPEC_TEMPLATE_FILE = "supervisor_spec_template.json";
-  private static final String SUPERVISOR_WITH_CONCURRENT_LOCKS_SPEC_TEMPLATE_FILE = "supervisor_with_concurrent_locks_spec_template.json";
   private static final String SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_FILE = "supervisor_with_autoscaler_spec_template.json";
   private static final String SUPERVISOR_WITH_IDLE_BEHAVIOUR_ENABLED_SPEC_TEMPLATE_FILE =
       "supervisor_with_idle_behaviour_enabled_spec_template.json";
@@ -89,8 +88,6 @@ public abstract class AbstractStreamIndexingTest extends AbstractIndexerTest
       "supervisor_with_long_duration.json";
 
   protected static final String DATA_RESOURCE_ROOT = "/stream/data";
-  protected static final String SUPERVISOR_WITH_CONCURRENT_LOCKS_SPEC_TEMPLATE_PATH =
-      String.join("/", DATA_RESOURCE_ROOT, SUPERVISOR_WITH_CONCURRENT_LOCKS_SPEC_TEMPLATE_FILE);
   protected static final String SUPERVISOR_SPEC_TEMPLATE_PATH =
       String.join("/", DATA_RESOURCE_ROOT, SUPERVISOR_SPEC_TEMPLATE_FILE);
   protected static final String SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_PATH =
diff --git a/integration-tests/src/test/resources/stream/data/supervisor_with_concurrent_locks_spec_template.json b/integration-tests/src/test/resources/stream/data/supervisor_with_concurrent_locks_spec_template.json
deleted file mode 100644
index 6adfc49da00..00000000000
--- a/integration-tests/src/test/resources/stream/data/supervisor_with_concurrent_locks_spec_template.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
-  "type": "%%STREAM_TYPE%%",
-  "dataSchema": {
-    "dataSource": "%%DATASOURCE%%",
-    "parser": %%PARSER%%,
-    "timestampSpec": {
-      "column": "timestamp",
-      "format": "auto"
-    },
-    "dimensionsSpec": {
-      "dimensions": %%DIMENSIONS%%,
-      "dimensionExclusions": [],
-      "spatialDimensions": []
-    },
-    "metricsSpec": [
-      {
-        "type": "count",
-        "name": "count"
-      },
-      {
-        "type": "doubleSum",
-        "name": "added",
-        "fieldName": "added"
-      },
-      {
-        "type": "doubleSum",
-        "name": "deleted",
-        "fieldName": "deleted"
-      },
-      {
-        "type": "doubleSum",
-        "name": "delta",
-        "fieldName": "delta"
-      }
-    ],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "MINUTE",
-      "queryGranularity": "NONE"
-    }
-  },
-  "tuningConfig": {
-    "type": "%%STREAM_TYPE%%",
-    "intermediatePersistPeriod": "PT30S",
-    "maxRowsPerSegment": 5000000,
-    "maxRowsInMemory": 500000
-  },
-  "ioConfig": {
-    "%%TOPIC_KEY%%": "%%TOPIC_VALUE%%",
-    "%%STREAM_PROPERTIES_KEY%%": %%STREAM_PROPERTIES_VALUE%%,
-    "taskCount": 2,
-    "replicas": 1,
-    "taskDuration": "PT120S",
-    "%%USE_EARLIEST_KEY%%": true,
-    "inputFormat" : %%INPUT_FORMAT%%
-  },
-  "context": {
-    "useConcurrentLocks": true
-  }
-}

From 9162339fa88892f04e0ed02778baa29807d4d350 Mon Sep 17 00:00:00 2001
From: Hugh Evans <101668034+hevansDev@users.noreply.github.com>
Date: Wed, 4 Sep 2024 20:45:58 +0100
Subject: [PATCH 88/99] Replace dsql instructions in example (#16977)

---
 docs/tutorials/tutorial-ingestion-spec.md | 32 +++++++++--------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/docs/tutorials/tutorial-ingestion-spec.md b/docs/tutorials/tutorial-ingestion-spec.md
index 5ac652fcbdb..947b73e2c7f 100644
--- a/docs/tutorials/tutorial-ingestion-spec.md
+++ b/docs/tutorials/tutorial-ingestion-spec.md
@@ -590,24 +590,18 @@ bin/post-index-task --file quickstart/ingestion-tutorial-index.json --url http:/
 
 After the script completes, we will query the data.
 
-Let's run `bin/dsql` and issue a `select * from "ingestion-tutorial";` query to see what data was ingested.
+In the web console, open a new tab in the **Query** view. Run the following query to view the ingested data:
 
-```bash
-$ bin/dsql
-Welcome to dsql, the command-line client for Druid SQL.
-Type "\h" for help.
-dsql> select * from "ingestion-tutorial";
-
-┌──────────────────────────┬───────┬──────┬───────┬─────────┬─────────┬─────────┬──────────┬─────────┬─────────┐
-│ __time                   │ bytes │ cost │ count │ dstIP   │ dstPort │ packets │ protocol │ srcIP   │ srcPort │
-├──────────────────────────┼───────┼──────┼───────┼─────────┼─────────┼─────────┼──────────┼─────────┼─────────┤
-│ 2018-01-01T01:01:00.000Z │  6000 │  4.9 │     3 │ 2.2.2.2 │    3000 │      60 │ 6        │ 1.1.1.1 │    2000 │
-│ 2018-01-01T01:02:00.000Z │  9000 │ 18.1 │     2 │ 2.2.2.2 │    7000 │      90 │ 6        │ 1.1.1.1 │    5000 │
-│ 2018-01-01T01:03:00.000Z │  6000 │  4.3 │     1 │ 2.2.2.2 │    7000 │      60 │ 6        │ 1.1.1.1 │    5000 │
-│ 2018-01-01T02:33:00.000Z │ 30000 │ 56.9 │     2 │ 8.8.8.8 │    5000 │     300 │ 17       │ 7.7.7.7 │    4000 │
-│ 2018-01-01T02:35:00.000Z │ 30000 │ 46.3 │     1 │ 8.8.8.8 │    5000 │     300 │ 17       │ 7.7.7.7 │    4000 │
-└──────────────────────────┴───────┴──────┴───────┴─────────┴─────────┴─────────┴──────────┴─────────┴─────────┘
-Retrieved 5 rows in 0.12s.
-
-dsql>
+```sql
+select * from "ingestion-tutorial"
 ```
+
+Returns the following:
+
+| `__time` | `bytes` | `cost` | `count` | `dstIP` | `dstPort` | `packets` | `protocol` | `srcIP` | `srcPort` |
+| -- | -- | -- | -- | -- | -- | -- | -- | -- | -- |
+| `2018-01-01T01:01:00.000Z` | `6000` | `4.9` | `3` | `2.2.2.2` | `3000` | `60` | `6` | `1.1.1.1` | `2000` |
+| `2018-01-01T01:02:00.000Z` |  `9000` | `18.1` | `2` | `2.2.2.2` | `7000` | `90` | `6` | `1.1.1.1` | `5000` |
+| `2018-01-01T01:03:00.000Z` | `6000` |  `4.3` | `1` | `2.2.2.2` | `7000` | `60` | `6` | `1.1.1.1` | `5000` |
+| `2018-01-01T02:33:00.000Z` | `30000` | `56.9` | `2` | `8.8.8.8` | `5000` | `300` | `17` | `7.7.7.7` | `4000` |
+| `2018-01-01T02:35:00.000Z` | `30000` | `46.3` | `1` | `8.8.8.8` | `5000` | `300` | `17` | `7.7.7.7` | `4000` |
\ No newline at end of file

From 4e02e5b8569832198464ebbbcf42fc5c9a903d95 Mon Sep 17 00:00:00 2001
From: Rishabh Singh <6513075+findingrish@users.noreply.github.com>
Date: Thu, 5 Sep 2024 07:58:24 +0530
Subject: [PATCH 89/99] Remove alert for pre-existing new columns while merging
 realtime schema (#16989)

Currently, an alert is thrown while merging datasource schema with realtime
segment schema when the datasource schema already has update columns from the delta schema.

This isn't an error condition since the datasource schema can have those columns from a different segment.

One scenario in which this can occur is when multiple replicas for a task is run.
---
 .../CoordinatorSegmentMetadataCache.java      | 29 +++++++++----------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
index 49254c1da5c..bfc7784caf3 100644
--- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
@@ -811,16 +811,14 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
         mergedColumnTypes.put(column, columnType);
       }
 
-      Map<String, ColumnType> columnMapping = segmentSchema.getColumnTypeMap();
+      final Map<String, ColumnType> columnMapping = segmentSchema.getColumnTypeMap();
 
       // column type to be updated is not present in the existing schema
-      boolean missingUpdateColumns = false;
-      // new column to be added is already present in the existing schema
-      boolean existingNewColumns = false;
+      final Set<String> missingUpdateColumns = new HashSet<>();
 
       for (String column : segmentSchema.getUpdatedColumns()) {
         if (!mergedColumnTypes.containsKey(column)) {
-          missingUpdateColumns = true;
+          missingUpdateColumns.add(column);
           mergedColumnTypes.put(column, columnMapping.get(column));
         } else {
           mergedColumnTypes.compute(column, (c, existingType) -> columnTypeMergePolicy.merge(existingType, columnMapping.get(column)));
@@ -829,23 +827,24 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
 
       for (String column : segmentSchema.getNewColumns()) {
         if (mergedColumnTypes.containsKey(column)) {
-          existingNewColumns = true;
           mergedColumnTypes.compute(column, (c, existingType) -> columnTypeMergePolicy.merge(existingType, columnMapping.get(column)));
         } else {
           mergedColumnTypes.put(column, columnMapping.get(column));
         }
       }
 
-      if (missingUpdateColumns || existingNewColumns) {
+      if (!missingUpdateColumns.isEmpty()) {
         log.makeAlert(
-            "Error merging delta schema update with existing row signature. segmentId [%s], "
-            + "existingSignature [%s], deltaSchema [%s], missingUpdateColumns [%s], existingNewColumns [%s].",
-            segmentId,
-            existingSignature,
-            segmentSchema,
-            missingUpdateColumns,
-            existingNewColumns
-        ).emit();
+               "Datasource schema mismatch detected. The delta realtime segment schema contains columns "
+               + "that are not defined in the datasource schema. "
+               + "This indicates a potential issue with schema updates on the Coordinator. "
+               + "Please review relevant Coordinator metrics and logs for task communication to identify any issues."
+           )
+           .addData("datasource", segmentId.getDataSource())
+           .addData("existingSignature", existingSignature)
+           .addData("deltaSchema", segmentSchema)
+           .addData("missingUpdateColumns", missingUpdateColumns)
+           .emit();
       }
 
       mergedColumnTypes.forEach(builder::add);

From 39161b0b23855d63b8a07778e37eb94e608f8c2b Mon Sep 17 00:00:00 2001
From: Rishabh Singh <6513075+findingrish@users.noreply.github.com>
Date: Thu, 5 Sep 2024 10:36:55 +0530
Subject: [PATCH 90/99] Use vault.centos.org to build Hadoop docker image
 (#16999)

The Dockerfile for building hadoop image is broken due to Centos 7 EOL.
Fixed it as per https://serverfault.com/a/1161847.
---
 examples/quickstart/tutorial/hadoop/docker/Dockerfile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/examples/quickstart/tutorial/hadoop/docker/Dockerfile b/examples/quickstart/tutorial/hadoop/docker/Dockerfile
index 7739e295545..9cada259cc1 100644
--- a/examples/quickstart/tutorial/hadoop/docker/Dockerfile
+++ b/examples/quickstart/tutorial/hadoop/docker/Dockerfile
@@ -19,6 +19,11 @@ FROM centos:7
 
 USER root
 
+# CentOS is EOL, have to use vault
+RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
+    sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \
+    sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+
 # install dev tools
 RUN yum clean all \
     && rpm --rebuilddb \

From 18a9a7570a9fa64eb007472850c4ce1d70f94ce8 Mon Sep 17 00:00:00 2001
From: Rishabh Singh <6513075+findingrish@users.noreply.github.com>
Date: Thu, 5 Sep 2024 11:00:25 +0530
Subject: [PATCH 91/99] Log a small subset of segments to refresh for debugging
 Coordinator refresh logic (#16998)

* Log a small number of segments to refresh per datasource in the Coordinator

* review comments

* Update log message
---
 .../metadata/AbstractSegmentMetadataCache.java        | 11 +++++++++++
 .../metadata/CoordinatorSegmentMetadataCache.java     |  7 +++++++
 2 files changed, 18 insertions(+)

diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
index a2d5999a0e1..d918ec5e3f2 100644
--- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java
@@ -761,6 +761,9 @@ public abstract class AbstractSegmentMetadataCache<T extends DataSourceInformati
     final Map<String, SegmentId> segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString);
 
     final Set<SegmentId> retVal = new HashSet<>();
+
+    logSegmentsToRefresh(dataSource, segments);
+
     final Sequence<SegmentAnalysis> sequence = runSegmentMetadataQuery(
         Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY)
     );
@@ -805,6 +808,14 @@ public abstract class AbstractSegmentMetadataCache<T extends DataSourceInformati
     return retVal;
   }
 
+  /**
+   * Log the segment details for a datasource to be refreshed for debugging purpose.
+   */
+  void logSegmentsToRefresh(String dataSource, Set<SegmentId> ids)
+  {
+    // no-op
+  }
+
   /**
    * Action to be executed on the result of Segment metadata query.
    * Returns if the segment metadata was updated.
diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
index bfc7784caf3..0ed81f6c1e8 100644
--- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
@@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Supplier;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import com.google.inject.Inject;
 import org.apache.druid.client.CoordinatorServerView;
@@ -529,6 +530,12 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
     }
   }
 
+  @Override
+  void logSegmentsToRefresh(String dataSource, Set<SegmentId> ids)
+  {
+    log.info("Logging a sample of 5 segments [%s] to be refreshed for datasource [%s]", Iterables.limit(ids, 5), dataSource);
+  }
+
   private void filterRealtimeSegments(Set<SegmentId> segmentIds)
   {
     synchronized (lock) {

From 40f38f0191dcd03b755224c2fb604ad732c7a8c7 Mon Sep 17 00:00:00 2001
From: Rishabh Singh <6513075+findingrish@users.noreply.github.com>
Date: Thu, 5 Sep 2024 16:07:33 +0530
Subject: [PATCH 92/99] Remove migrated deep storage standard ITs  (#16933)

---
 .../ITAzureToAzureParallelIndexTest.java      | 49 ------------------
 .../indexer/ITGcsToGcsParallelIndexTest.java  | 50 -------------------
 .../indexer/ITS3ToS3ParallelIndexTest.java    | 49 ------------------
 3 files changed, 148 deletions(-)
 delete mode 100644 integration-tests/src/test/java/org/apache/druid/tests/indexer/ITAzureToAzureParallelIndexTest.java
 delete mode 100644 integration-tests/src/test/java/org/apache/druid/tests/indexer/ITGcsToGcsParallelIndexTest.java
 delete mode 100644 integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3ToS3ParallelIndexTest.java

diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITAzureToAzureParallelIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITAzureToAzureParallelIndexTest.java
deleted file mode 100644
index cbb8bba877a..00000000000
--- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITAzureToAzureParallelIndexTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.tests.indexer;
-
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.tests.TestNGGroup;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import java.util.List;
-
-/**
- * IMPORTANT:
- * To run this test, you must:
- * 1) Set the bucket and path for your data. This can be done by setting -Ddruid.test.config.cloudBucket and
- *    -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path" in the config file.
- * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and wikipedia_index_data3.json
- *    located in integration-tests/src/test/resources/data/batch_index/json to your Azure at the location set in step 1.
- * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with Azure credentials/configs set. See
- *    integration-tests/docker/environment-configs/override-examples/azure for env vars to provide.
- */
-@Test(groups = TestNGGroup.AZURE_DEEP_STORAGE)
-@Guice(moduleFactory = DruidTestModuleFactory.class)
-public class ITAzureToAzureParallelIndexTest extends AbstractAzureInputSourceParallelIndexTest
-{
-  @Test(dataProvider = "resources")
-  public void testAzureIndexData(Pair<String, List> azureInputSource) throws Exception
-  {
-    doTest(azureInputSource, new Pair<>(false, false));
-  }
-}
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITGcsToGcsParallelIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITGcsToGcsParallelIndexTest.java
deleted file mode 100644
index 927cb3ee281..00000000000
--- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITGcsToGcsParallelIndexTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.tests.indexer;
-
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.tests.TestNGGroup;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import java.util.List;
-
-/**
- * IMPORTANT:
- * To run this test, you must:
- * 1) Set the bucket and path for your data. This can be done by setting -Ddruid.test.config.cloudBucket and
- *    -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path" in the config file.
- * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and wikipedia_index_data3.json
- *    located in integration-tests/src/test/resources/data/batch_index/json to your GCS at the location set in step 1.
- * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with gcs configs set. See
- *    integration-tests/docker/environment-configs/override-examples/gcs for env vars to provide.
- * 4) Provide -Dresource.file.dir.path=<PATH_TO_FOLDER> with folder that contains GOOGLE_APPLICATION_CREDENTIALS file
- */
-@Test(groups = TestNGGroup.GCS_DEEP_STORAGE)
-@Guice(moduleFactory = DruidTestModuleFactory.class)
-public class ITGcsToGcsParallelIndexTest extends AbstractGcsInputSourceParallelIndexTest
-{
-  @Test(dataProvider = "resources")
-  public void testGcsIndexData(Pair<String, List> gcsInputSource) throws Exception
-  {
-    doTest(gcsInputSource, new Pair<>(false, false));
-  }
-}
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3ToS3ParallelIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3ToS3ParallelIndexTest.java
deleted file mode 100644
index 0a85d38fe16..00000000000
--- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3ToS3ParallelIndexTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.tests.indexer;
-
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.tests.TestNGGroup;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import java.util.List;
-
-/**
- * IMPORTANT:
- * To run this test, you must:
- * 1) Set the bucket and path for your data. This can be done by setting -Ddruid.test.config.cloudBucket and
- *    -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path" in the config file.
- * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and wikipedia_index_data3.json
- *    located in integration-tests/src/test/resources/data/batch_index/json to your S3 at the location set in step 1.
- * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with s3 credentials/configs set. See
- *    integration-tests/docker/environment-configs/override-examples/s3 for env vars to provide.
- */
-@Test(groups = TestNGGroup.S3_DEEP_STORAGE)
-@Guice(moduleFactory = DruidTestModuleFactory.class)
-public class ITS3ToS3ParallelIndexTest extends AbstractS3InputSourceParallelIndexTest
-{
-  @Test(dataProvider = "resources")
-  public void testS3IndexData(Pair<String, List> s3InputSource) throws Exception
-  {
-    doTest(s3InputSource, new Pair<>(false, false));
-  }
-}

From b4d83a86c20f55f85509d2a1f53925b6715a6b82 Mon Sep 17 00:00:00 2001
From: Jill Osborne <jill.osborne@imply.io>
Date: Thu, 5 Sep 2024 18:25:30 +0100
Subject: [PATCH 93/99] Middle Manager wording update in docs (#17005)

---
 docs/api-reference/service-status-api.md      | 24 ++---
 docs/configuration/index.md                   | 94 +++++++++----------
 docs/design/architecture.md                   | 28 +++---
 docs/design/indexer.md                        |  8 +-
 docs/design/indexing-service.md               |  4 +-
 docs/design/metadata-storage.md               |  2 +-
 docs/design/middlemanager.md                  | 14 +--
 docs/design/overlord.md                       | 10 +-
 docs/design/peons.md                          |  8 +-
 docs/design/storage.md                        |  4 +-
 docs/design/zookeeper.md                      |  2 +-
 .../extensions-contrib/gce-extensions.md      |  4 +-
 .../extensions-contrib/prometheus.md          |  2 +-
 docs/development/extensions-core/s3.md        |  2 +-
 docs/ingestion/hadoop.md                      |  8 +-
 docs/ingestion/index.md                       |  2 +-
 docs/ingestion/input-sources.md               |  2 +-
 docs/ingestion/kafka-ingestion.md             |  4 +-
 docs/ingestion/kinesis-ingestion.md           |  4 +-
 docs/ingestion/native-batch.md                | 10 +-
 docs/ingestion/streaming.md                   |  2 +-
 docs/ingestion/supervisor.md                  |  2 +-
 docs/ingestion/tasks.md                       | 12 +--
 docs/operations/basic-cluster-tuning.md       | 28 +++---
 docs/operations/other-hadoop.md               |  2 +-
 docs/operations/rolling-updates.md            | 10 +-
 docs/querying/groupbyquery.md                 |  2 +-
 docs/querying/query-execution.md              |  2 +-
 docs/querying/query-processing.md             |  4 +-
 docs/release-info/upgrade-notes.md            |  2 +-
 docs/tutorials/cluster.md                     | 18 ++--
 31 files changed, 160 insertions(+), 160 deletions(-)

diff --git a/docs/api-reference/service-status-api.md b/docs/api-reference/service-status-api.md
index 59e2b19c3fb..efa5ebe01c9 100644
--- a/docs/api-reference/service-status-api.md
+++ b/docs/api-reference/service-status-api.md
@@ -45,7 +45,7 @@ You can use each endpoint with the ports for each type of service. The following
 | Router|8888|
 | Broker|8082|
 | Historical|8083|
-| MiddleManager|8091|
+| Middle Manager|8091|
 
 ### Get service information
 
@@ -791,11 +791,11 @@ Host: http://OVERLORD_IP:OVERLORD_PORT
 </details>
 
 
-## MiddleManager
+## Middle Manager
 
-### Get MiddleManager state status
+### Get Middle Manager state status
 
-Retrieves the enabled state of the MiddleManager. Returns JSON object keyed by the combined `druid.host` and `druid.port` with a boolean `true` or `false` state as the value.
+Retrieves the enabled state of the Middle Manager process. Returns JSON object keyed by the combined `druid.host` and `druid.port` with a boolean `true` or `false` state as the value.
 
 #### URL
 
@@ -810,7 +810,7 @@ Retrieves the enabled state of the MiddleManager. Returns JSON object keyed by t
 
 <br/>
 
-*Successfully retrieved MiddleManager state*
+*Successfully retrieved Middle Manager state*
 
 </TabItem>
 </Tabs>
@@ -855,7 +855,7 @@ Host: http://MIDDLEMANAGER_IP:MIDDLEMANAGER_PORT
 
 ### Get active tasks
 
-Retrieves a list of active tasks being run on MiddleManager. Returns JSON list of task ID strings. Note that for normal usage, you should use the `/druid/indexer/v1/tasks` [Tasks API](./tasks-api.md) endpoint or one of the task state specific variants instead.
+Retrieves a list of active tasks being run on the Middle Manager. Returns JSON list of task ID strings. Note that for normal usage, you should use the `/druid/indexer/v1/tasks` [Tasks API](./tasks-api.md) endpoint or one of the task state specific variants instead.
 
 #### URL
 
@@ -984,9 +984,9 @@ Host: http://MIDDLEMANAGER_IP:MIDDLEMANAGER_PORT
 
 </details>
 
-### Disable MiddleManager
+### Disable Middle Manager
 
-Disables a MiddleManager, causing it to stop accepting new tasks but complete all existing tasks. Returns a JSON  object
+Disables a Middle Manager, causing it to stop accepting new tasks but complete all existing tasks. Returns a JSON  object
 keyed by the combined `druid.host` and `druid.port`.
 
 #### URL
@@ -1002,7 +1002,7 @@ keyed by the combined `druid.host` and `druid.port`.
 
 <br/>
 
-*Successfully disabled MiddleManager*
+*Successfully disabled Middle Manager*
 
 </TabItem>
 </Tabs>
@@ -1043,9 +1043,9 @@ Host: http://MIDDLEMANAGER_IP:MIDDLEMANAGER_PORT
 
 </details>
 
-### Enable MiddleManager
+### Enable Middle Manager
 
-Enables a MiddleManager, allowing it to accept new tasks again if it was previously disabled. Returns a JSON object keyed by the combined `druid.host` and `druid.port`.
+Enables a Middle Manager, allowing it to accept new tasks again if it was previously disabled. Returns a JSON object keyed by the combined `druid.host` and `druid.port`.
 
 #### URL
 
@@ -1060,7 +1060,7 @@ Enables a MiddleManager, allowing it to accept new tasks again if it was previou
 
 <br/>
 
-*Successfully enabled MiddleManager*
+*Successfully enabled Middle Manager*
 
 </TabItem>
 </Tabs>
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 9c83946eb82..64797c402c0 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -166,8 +166,8 @@ The indexing service also uses its own set of paths. These configs can be includ
 |Property|Description|Default|
 |--------|-----------|-------|
 |`druid.zk.paths.indexer.base`|Base ZooKeeper path for |`${druid.zk.paths.base}/indexer`|
-|`druid.zk.paths.indexer.announcementsPath`|MiddleManagers announce themselves here.|`${druid.zk.paths.indexer.base}/announcements`|
-|`druid.zk.paths.indexer.tasksPath`|Used to assign tasks to MiddleManagers.|`${druid.zk.paths.indexer.base}/tasks`|
+|`druid.zk.paths.indexer.announcementsPath`|Middle Managers announce themselves here.|`${druid.zk.paths.indexer.base}/announcements`|
+|`druid.zk.paths.indexer.tasksPath`|Used to assign tasks to Middle Managers.|`${druid.zk.paths.indexer.base}/tasks`|
 |`druid.zk.paths.indexer.statusPath`|Parent path for announcement of task statuses.|`${druid.zk.paths.indexer.base}/status`|
 
 If `druid.zk.paths.base` and `druid.zk.paths.indexer.base` are both set, and none of the other `druid.zk.paths.*` or `druid.zk.paths.indexer.*` values are set, then the other properties will be evaluated relative to their respective `base`.
@@ -403,7 +403,7 @@ Metric monitoring is an essential part of Druid operations. The following monito
 |`org.apache.druid.server.emitter.HttpEmittingMonitor`|Reports internal metrics of `http` or `parametrized` emitter (see below). Must not be used with another emitter type. See the description of the metrics here: <https://github.com/apache/druid/pull/4973>.|
 |`org.apache.druid.server.metrics.TaskCountStatsMonitor`|Reports how many ingestion tasks are currently running/pending/waiting and also the number of successful/failed tasks per emission period.|
 |`org.apache.druid.server.metrics.TaskSlotCountStatsMonitor`|Reports metrics about task slot usage per emission period.|
-|`org.apache.druid.server.metrics.WorkerTaskCountStatsMonitor`|Reports how many ingestion tasks are currently running/pending/waiting, the number of successful/failed tasks, and metrics about task slot usage for the reporting worker, per emission period. Only supported by MiddleManager node types.|
+|`org.apache.druid.server.metrics.WorkerTaskCountStatsMonitor`|Reports how many ingestion tasks are currently running/pending/waiting, the number of successful/failed tasks, and metrics about task slot usage for the reporting worker, per emission period. Only supported by Middle Manager node types.|
 |`org.apache.druid.server.metrics.ServiceStatusMonitor`|Reports a heartbeat for the service.|
 
 For example, you might configure monitors on all services for system and JVM information within `common.runtime.properties` as follows:
@@ -594,7 +594,7 @@ need arises.
 |Property|Description|Default|Required|
 |-----|-----------|-------|--------|
 |`druid.centralizedDatasourceSchema.enabled`|Boolean flag for enabling datasource schema building in the Coordinator, this should be specified in the common runtime properties.|false|No.|
-|`druid.indexer.fork.property.druid.centralizedDatasourceSchema.enabled`| This config should be set when CentralizedDatasourceSchema feature is enabled. This should be specified in the MiddleManager runtime properties.|false|No.|
+|`druid.indexer.fork.property.druid.centralizedDatasourceSchema.enabled`| This config should be set when CentralizedDatasourceSchema feature is enabled. This should be specified in the Middle Manager runtime properties.|false|No.|
 
 If you enable this feature, you can query datasources that are only stored in deep storage and are not loaded on a Historical. For more information, see [Query from deep storage](../querying/query-from-deep-storage.md).
 
@@ -1132,17 +1132,17 @@ The following configs only apply if the Overlord is running in remote mode. For
 
 |Property|Description|Default|
 |--------|-----------|-------|
-|`druid.indexer.runner.taskAssignmentTimeout`|How long to wait after a task has been assigned to a MiddleManager before throwing an error.|`PT5M`|
-|`druid.indexer.runner.minWorkerVersion`|The minimum MiddleManager version to send tasks to. The version number is a string. This affects the expected behavior during certain operations like comparison against `druid.worker.version`. Specifically, the version comparison follows dictionary order. Use ISO8601 date format for the version to accommodate date comparisons. |"0"|
+|`druid.indexer.runner.taskAssignmentTimeout`|How long to wait after a task has been assigned to a Middle Manager before throwing an error.|`PT5M`|
+|`druid.indexer.runner.minWorkerVersion`|The minimum Middle Manager version to send tasks to. The version number is a string. This affects the expected behavior during certain operations like comparison against `druid.worker.version`. Specifically, the version comparison follows dictionary order. Use ISO8601 date format for the version to accommodate date comparisons. |"0"|
 | `druid.indexer.runner.parallelIndexTaskSlotRatio`| The ratio of task slots available for parallel indexing supervisor tasks per worker. The specified value must be in the range `[0, 1]`. |1|
-|`druid.indexer.runner.compressZnodes`|Indicates whether or not the Overlord should expect MiddleManagers to compress Znodes.|true|
+|`druid.indexer.runner.compressZnodes`|Indicates whether or not the Overlord should expect Middle Managers to compress Znodes.|true|
 |`druid.indexer.runner.maxZnodeBytes`|The maximum size Znode in bytes that can be created in ZooKeeper, should be in the range of `[10KiB, 2GiB)`. [Human-readable format](human-readable-byte.md) is supported.| 512 KiB |
-|`druid.indexer.runner.taskCleanupTimeout`|How long to wait before failing a task after a MiddleManager is disconnected from ZooKeeper.|`PT15M`|
-|`druid.indexer.runner.taskShutdownLinkTimeout`|How long to wait on a shutdown request to a MiddleManager before timing out|`PT1M`|
+|`druid.indexer.runner.taskCleanupTimeout`|How long to wait before failing a task after a Middle Manager is disconnected from ZooKeeper.|`PT15M`|
+|`druid.indexer.runner.taskShutdownLinkTimeout`|How long to wait on a shutdown request to a Middle Manager before timing out|`PT1M`|
 |`druid.indexer.runner.pendingTasksRunnerNumThreads`|Number of threads to allocate pending-tasks to workers, must be at least 1.|1|
-|`druid.indexer.runner.maxRetriesBeforeBlacklist`|Number of consecutive times the MiddleManager can fail tasks,  before the worker is blacklisted, must be at least 1|5|
+|`druid.indexer.runner.maxRetriesBeforeBlacklist`|Number of consecutive times the Middle Manager can fail tasks,  before the worker is blacklisted, must be at least 1|5|
 |`druid.indexer.runner.workerBlackListBackoffTime`|How long to wait before a task is whitelisted again. This value should be greater that the value set for taskBlackListCleanupPeriod.|`PT15M`|
-|`druid.indexer.runner.workerBlackListCleanupPeriod`|A duration after which the cleanup thread will startup to clean blacklisted workers.|`PT5M`|
+|`druid.indexer.runner.workerBlackListCleanupPeriod`|A duration after which the cleanup thread will start up to clean blacklisted workers.|`PT5M`|
 |`druid.indexer.runner.maxPercentageBlacklistWorkers`|The maximum percentage of workers to blacklist, this must be between 0 and 100.|20|
 
 If autoscaling is enabled, you can set these additional configs:
@@ -1151,16 +1151,16 @@ If autoscaling is enabled, you can set these additional configs:
 |--------|-----------|-------|
 |`druid.indexer.autoscale.strategy`|Sets the strategy to run when autoscaling is required. One of `noop`, `ec2` or `gce`.|`noop`|
 |`druid.indexer.autoscale.doAutoscale`|If set to true, autoscaling will be enabled.|false|
-|`druid.indexer.autoscale.provisionPeriod`|How often to check whether or not new MiddleManagers should be added.|`PT1M`|
-|`druid.indexer.autoscale.terminatePeriod`|How often to check when MiddleManagers should be removed.|`PT5M`|
+|`druid.indexer.autoscale.provisionPeriod`|How often to check whether or not new Middle Managers should be added.|`PT1M`|
+|`druid.indexer.autoscale.terminatePeriod`|How often to check when Middle Managers should be removed.|`PT5M`|
 |`druid.indexer.autoscale.originTime`|The starting reference timestamp that the terminate period increments upon.|`2012-01-01T00:55:00.000Z`|
 |`druid.indexer.autoscale.workerIdleTimeout`|How long can a worker be idle (not a run task) before it can be considered for termination.|`PT90M`|
-|`druid.indexer.autoscale.maxScalingDuration`|How long the Overlord will wait around for a MiddleManager to show up before giving up.|`PT15M`|
+|`druid.indexer.autoscale.maxScalingDuration`|How long the Overlord will wait around for a Middle Manager to show up before giving up.|`PT15M`|
 |`druid.indexer.autoscale.numEventsToTrack`|The number of autoscaling related events (node creation and termination) to track.|10|
 |`druid.indexer.autoscale.pendingTaskTimeout`|How long a task can be in "pending" state before the Overlord tries to scale up.|`PT30S`|
 |`druid.indexer.autoscale.workerVersion`|If set, will only create nodes of set version during autoscaling. Overrides dynamic configuration. |null|
-|`druid.indexer.autoscale.workerPort`|The port that MiddleManagers will run on.|8080|
-|`druid.indexer.autoscale.workerCapacityHint`| An estimation of the number of task slots available for each worker launched by the auto scaler when there are no workers running. The auto scaler uses the worker capacity hint to launch workers with an adequate capacity to handle pending tasks. When unset or set to a value less than or equal to 0, the auto scaler scales workers equal to the value for `minNumWorkers` in autoScaler config instead. The auto scaler assumes that each worker, either a MiddleManager or indexer, has the same amount of task slots. Therefore, when all your workers have the same capacity (homogeneous capacity), set the value for `autoscale.workerCapacityHint` equal to `druid.worker.capacity`. If your workers have different capacities (heterogeneous capacity), set the value to the average of `druid.worker.capacity` across the workers. For example, if two workers have `druid.worker.capacity=10`, and one has `druid.worker.capacity=4`, set `autoscale.workerCapacityHint=8`. Only applies to `pendingTaskBased` provisioning strategy.|-1|
+|`druid.indexer.autoscale.workerPort`|The port that Middle Managers will run on.|8080|
+|`druid.indexer.autoscale.workerCapacityHint`| An estimation of the number of task slots available for each worker launched by the auto scaler when there are no workers running. The auto scaler uses the worker capacity hint to launch workers with an adequate capacity to handle pending tasks. When unset or set to a value less than or equal to 0, the auto scaler scales workers equal to the value for `minNumWorkers` in autoScaler config instead. The auto scaler assumes that each worker, either a Middle Manager or indexer, has the same amount of task slots. Therefore, when all your workers have the same capacity (homogeneous capacity), set the value for `autoscale.workerCapacityHint` equal to `druid.worker.capacity`. If your workers have different capacities (heterogeneous capacity), set the value to the average of `druid.worker.capacity` across the workers. For example, if two workers have `druid.worker.capacity=10`, and one has `druid.worker.capacity=4`, set `autoscale.workerCapacityHint=8`. Only applies to `pendingTaskBased` provisioning strategy.|-1|
 
 ##### Supervisors
 
@@ -1186,7 +1186,7 @@ The following table shows the dynamic configuration properties for the Overlord.
 
 |Property|Description|Default|
 |--------|-----------|-------|
-|`selectStrategy`| Describes how to assign tasks to MiddleManagers. The type can be `equalDistribution`, `equalDistributionWithCategorySpec`, `fillCapacity`, `fillCapacityWithCategorySpec`, and `javascript`. | `{"type":"equalDistribution"}` |
+|`selectStrategy`| Describes how to assign tasks to Middle Managers. The type can be `equalDistribution`, `equalDistributionWithCategorySpec`, `fillCapacity`, `fillCapacityWithCategorySpec`, and `javascript`. | `{"type":"equalDistribution"}` |
 |`autoScaler`| Only used if [autoscaling](#autoscaler) is enabled.| null |
 
 The following is an example of an Overlord dynamic config:
@@ -1233,7 +1233,7 @@ The following is an example of an Overlord dynamic config:
 
 ##### Worker select strategy
 
-The select strategy controls how Druid assigns tasks to workers (MiddleManagers).
+The select strategy controls how Druid assigns tasks to workers (Middle Managers).
 At a high level, the select strategy determines the list of eligible workers for a given task using
 either an `affinityConfig` or a `categorySpec`. Then, Druid assigns the task by either trying to distribute load equally
 (`equalDistribution`) or to fill as many workers as possible to capacity (`fillCapacity`).
@@ -1270,8 +1270,8 @@ not be assigned a category, and you want the work to be concentrated on the fewe
 
 ###### `equalDistribution`
 
-Tasks are assigned to the MiddleManager with the most free slots at the time the task begins running.
-This evenly distributes work across your MiddleManagers.
+Tasks are assigned to the Middle Manager with the most free slots at the time the task begins running.
+This evenly distributes work across your Middle Managers.
 
 |Property|Description|Default|
 |--------|-----------|-------|
@@ -1281,7 +1281,7 @@ This evenly distributes work across your MiddleManagers.
 ###### `equalDistributionWithCategorySpec`
 
 This strategy is a variant of `equalDistribution`, which supports `workerCategorySpec` field rather than `affinityConfig`.
-By specifying `workerCategorySpec`, you can assign tasks to run on different categories of MiddleManagers based on the **type** and **dataSource** of the task.
+By specifying `workerCategorySpec`, you can assign tasks to run on different categories of Middle Managers based on the **type** and **dataSource** of the task.
 This strategy doesn't work with `AutoScaler` since the behavior is undefined.
 
 |Property|Description|Default|
@@ -1289,7 +1289,7 @@ This strategy doesn't work with `AutoScaler` since the behavior is undefined.
 |`type`|`equalDistributionWithCategorySpec`|required; must be `equalDistributionWithCategorySpec`|
 |`workerCategorySpec`|[`WorkerCategorySpec`](#workercategoryspec) object|null (no worker category spec)|
 
-The following example shows tasks of type `index_kafka` that default to running on MiddleManagers of category `c1`, except for tasks that write to datasource `ds1`, which run on MiddleManagers of category `c2`.
+The following example shows tasks of type `index_kafka` that default to running on Middle Managers of category `c1`, except for tasks that write to datasource `ds1`, which run on Middle Managers of category `c2`.
 
 ```json
 {
@@ -1313,11 +1313,11 @@ The following example shows tasks of type `index_kafka` that default to running
 ###### `fillCapacity`
 
 Tasks are assigned to the worker with the most currently-running tasks. This is
-useful when you are auto-scaling MiddleManagers since it tends to pack some full and
+useful when you are auto-scaling Middle Managers since it tends to pack some full and
 leave others empty. The empty ones can be safely terminated.
 
 Note that if `druid.indexer.runner.pendingTasksRunnerNumThreads` is set to _N_ > 1, then this strategy will fill _N_
-MiddleManagers up to capacity simultaneously, rather than a single MiddleManager.
+Middle Managers up to capacity simultaneously, rather than a single Middle Manager.
 
 |Property|Description|Default|
 |--------|-----------|-------|
@@ -1370,8 +1370,8 @@ If not provided, the default is to have no affinity.
 
 |Property|Description|Default|
 |--------|-----------|-------|
-|`affinity`|JSON object mapping a datasource String name to a list of indexing service MiddleManager `host:port` values. Druid doesn't perform DNS resolution, so the 'host' value must match what is configured on the MiddleManager and what the MiddleManager announces itself as (examine the Overlord logs to see what your MiddleManager announces itself as).|`{}`|
-|`strong`|When `true` tasks for a datasource must be assigned to affinity-mapped MiddleManagers. Tasks remain queued until a slot becomes available. When `false`, Druid may assign tasks for a datasource to other MiddleManagers when affinity-mapped MiddleManagers are unavailable to run queued tasks.|false|
+|`affinity`|JSON object mapping a datasource String name to a list of indexing service Middle Manager `host:port` values. Druid doesn't perform DNS resolution, so the 'host' value must match what is configured on the Middle Manager and what the Middle Manager announces itself as (examine the Overlord logs to see what your Middle Manager announces itself as).|`{}`|
+|`strong`|When `true` tasks for a datasource must be assigned to affinity-mapped Middle Managers. Tasks remain queued until a slot becomes available. When `false`, Druid may assign tasks for a datasource to other Middle Managers when affinity-mapped Middle Managers are unavailable to run queued tasks.|false|
 
 ###### workerCategorySpec
 
@@ -1381,14 +1381,14 @@ field. If not provided, the default is to not use it at all.
 |Property|Description|Default|
 |--------|-----------|-------|
 |`categoryMap`|A JSON map object mapping a task type String name to a [CategoryConfig](#categoryconfig) object, by which you can specify category config for different task type.|`{}`|
-|`strong`|With weak workerCategorySpec (the default), tasks for a dataSource may be assigned to other MiddleManagers if the MiddleManagers specified in `categoryMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerCategorySpec, tasks for a dataSource will only ever be assigned to their specified MiddleManagers, and will wait in the pending queue if necessary.|false|
+|`strong`|With weak workerCategorySpec (the default), tasks for a dataSource may be assigned to other Middle Managers if the Middle Managers specified in `categoryMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerCategorySpec, tasks for a dataSource will only ever be assigned to their specified Middle Managers, and will wait in the pending queue if necessary.|false|
 
 ###### CategoryConfig
 
 |Property|Description|Default|
 |--------|-----------|-------|
 |`defaultCategory`|Specify default category for a task type.|null|
-|`categoryAffinity`|A JSON map object mapping a datasource String name to a category String name of the MiddleManager. If category isn't specified for a datasource, then using the `defaultCategory`. If no specified category and the `defaultCategory` is also null, then tasks can run on any available MiddleManagers.|null|
+|`categoryAffinity`|A JSON map object mapping a datasource String name to a category String name of the Middle Manager. If category isn't specified for a datasource, then using the `defaultCategory`. If no specified category and the `defaultCategory` is also null, then tasks can run on any available Middle Managers.|null|
 
 ##### Autoscaler
 
@@ -1409,15 +1409,15 @@ For GCE's properties, please refer to the [gce-extensions](../development/extens
 
 ## Data server
 
-This section contains the configuration options for the services that reside on Data servers (MiddleManagers/Peons and Historicals) in the suggested [three-server configuration](../design/architecture.md#druid-servers).
+This section contains the configuration options for the services that reside on Data servers (Middle Managers/Peons and Historicals) in the suggested [three-server configuration](../design/architecture.md#druid-servers).
 
 Configuration options for the [Indexer process](../design/indexer.md) are also provided here.
 
-### MiddleManager and Peons
+### Middle Manager and Peon
 
-These MiddleManager and Peon configurations can be defined in the `middleManager/runtime.properties` file.
+These Middle Manager and Peon configurations can be defined in the `middleManager/runtime.properties` file.
 
-#### MiddleManager service config
+#### Middle Manager service config
 
 |Property|Description|Default|
 |--------|-----------|-------|
@@ -1427,14 +1427,14 @@ These MiddleManager and Peon configurations can be defined in the `middleManager
 |`druid.tlsPort`|TLS port for HTTPS connector, if [druid.enableTlsPort](../operations/tls-support.md) is set then this config will be used. If `druid.host` contains port then that port will be ignored. This should be a non-negative Integer.|8291|
 |`druid.service`|The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services|`druid/middlemanager`|
 
-#### MiddleManager configuration
+#### Middle Manager configuration
 
-MiddleManagers pass their configurations down to their child peons. The MiddleManager requires the following configs:
+Middle Managers pass their configurations down to their child peons. The Middle Manager requires the following configs:
 
 |Property|Description|Default|
 |--------|-----------|-------|
 |`druid.indexer.runner.allowedPrefixes`|Whitelist of prefixes for configs that can be passed down to child peons.|`com.metamx`, `druid`, `org.apache.druid`, `user.timezone`, `file.encoding`, `java.io.tmpdir`, `hadoop`|
-|`druid.indexer.runner.compressZnodes`|Indicates whether or not the MiddleManagers should compress Znodes.|true|
+|`druid.indexer.runner.compressZnodes`|Indicates whether or not the Middle Managers should compress Znodes.|true|
 |`druid.indexer.runner.classpath`|Java classpath for the peon.|`System.getProperty("java.class.path")`|
 |`druid.indexer.runner.javaCommand`|Command required to execute java.|java|
 |`druid.indexer.runner.javaOpts`|_DEPRECATED_ A string of -X Java options to pass to the peon's JVM. Quotable parameters or parameters with spaces are encouraged to use javaOptsArray|`''`|
@@ -1444,16 +1444,16 @@ MiddleManagers pass their configurations down to their child peons. The MiddleMa
 |`druid.indexer.runner.endPort`|Ending port used for Peon services, should be greater than or equal to `druid.indexer.runner.startPort` and less than 65536.|65535|
 |`druid.indexer.runner.ports`|A JSON array of integers to specify ports that used for Peon services. If provided and non-empty, ports for Peon services will be chosen from these ports. And `druid.indexer.runner.startPort/druid.indexer.runner.endPort` will be completely ignored.|`[]`|
 |`druid.worker.ip`|The IP of the worker.|`localhost`|
-|`druid.worker.version`|Version identifier for the MiddleManager. The version number is a string. This affects the expected behavior during certain operations like comparison against `druid.indexer.runner.minWorkerVersion`. Specifically, the version comparison follows dictionary order. Use ISO8601 date format for the version to accommodate date comparisons.|0|
-|`druid.worker.capacity`|Maximum number of tasks the MiddleManager can accept.|Number of CPUs on the machine - 1|
+|`druid.worker.version`|Version identifier for the Middle Manager. The version number is a string. This affects the expected behavior during certain operations like comparison against `druid.indexer.runner.minWorkerVersion`. Specifically, the version comparison follows dictionary order. Use ISO8601 date format for the version to accommodate date comparisons.|0|
+|`druid.worker.capacity`|Maximum number of tasks the Middle Manager can accept.|Number of CPUs on the machine - 1|
 |`druid.worker.baseTaskDirs`|List of base temporary working directories, one of which is assigned per task in a round-robin fashion. This property can be used to allow usage of multiple disks for indexing. This property is recommended in place of and takes precedence over `${druid.indexer.task.baseTaskDir}`.  If this configuration is not set, `${druid.indexer.task.baseTaskDir}` is used. For example, `druid.worker.baseTaskDirs=[\"PATH1\",\"PATH2\",...]`.|null|
 |`druid.worker.baseTaskDirSize`|The total amount of bytes that can be used by tasks on any single task dir. This value is treated symmetrically across all directories, that is, if this is 500 GB and there are 3 `baseTaskDirs`, then each of those task directories is assumed to allow for 500 GB to be used and a total of 1.5 TB will potentially be available across all tasks. The actual amount of memory assigned to each task is discussed in [Configuring task storage sizes](../ingestion/tasks.md#configuring-task-storage-sizes)|`Long.MAX_VALUE`|
-|`druid.worker.category`|A string to name the category that the MiddleManager node belongs to.|`_default_worker_category`|
+|`druid.worker.category`|A string to name the category that the Middle Manager node belongs to.|`_default_worker_category`|
 |`druid.indexer.fork.property.druid.centralizedDatasourceSchema.enabled`| This config should be set when [Centralized Datasource Schema](#centralized-datasource-schema) feature is enabled. |false|
 
 #### Peon processing
 
-Processing properties set on the MiddleManager are passed through to Peons.
+Processing properties set on the Middle Manager are passed through to Peons.
 
 |Property|Description|Default|
 |--------|-----------|-------|
@@ -1464,7 +1464,7 @@ Processing properties set on the MiddleManager are passed through to Peons.
 |`druid.processing.numThreads`|The number of processing threads to have available for parallel processing of segments. Our rule of thumb is `num_cores - 1`, which means that even under heavy load there will still be one core available to do background tasks like talking with ZooKeeper and pulling down segments. If only one core is available, this property defaults to the value `1`.|Number of cores - 1 (or 1)|
 |`druid.processing.fifo`|Enables the processing queue to treat tasks of equal priority in a FIFO manner.|`true`|
 |`druid.processing.tmpDir`|Path where temporary files created while processing a query should be stored. If specified, this configuration takes priority over the default `java.io.tmpdir` path.|path represented by `java.io.tmpdir`|
-|`druid.processing.intermediaryData.storage.type`|Storage type for intermediary segments of data shuffle between native parallel index tasks. <br />Set to `local` to store segment files in the local storage of the MiddleManager or Indexer. <br />Set to `deepstore` to use configured deep storage for better fault tolerance during rolling updates. When the storage type is `deepstore`, Druid stores the data in the `shuffle-data` directory under the configured deep storage path. Druid does not support automated cleanup for the `shuffle-data` directory. You can set up cloud storage lifecycle rules for automated cleanup of data at the `shuffle-data` prefix location.|`local`|
+|`druid.processing.intermediaryData.storage.type`|Storage type for intermediary segments of data shuffle between native parallel index tasks. <br />Set to `local` to store segment files in the local storage of the Middle Manager or Indexer. <br />Set to `deepstore` to use configured deep storage for better fault tolerance during rolling updates. When the storage type is `deepstore`, Druid stores the data in the `shuffle-data` directory under the configured deep storage path. Druid does not support automated cleanup for the `shuffle-data` directory. You can set up cloud storage lifecycle rules for automated cleanup of data at the `shuffle-data` prefix location.|`local`|
 
 The amount of direct memory needed by Druid is at least
 `druid.processing.buffer.sizeBytes * (druid.processing.numMergeBuffers + druid.processing.numThreads + 1)`. You can
@@ -1490,7 +1490,7 @@ See [cache configuration](#cache-configuration) for how to configure cache setti
 
 #### Additional Peon configuration
 
-Although Peons inherit the configurations of their parent MiddleManagers, explicit child Peon configs in MiddleManager can be set by prefixing them with:
+Although Peons inherit the configurations of their parent Middle Managers, explicit child Peon configs in Middle Manager can be set by prefixing them with:
 
 ```properties
 druid.indexer.fork.property
@@ -1506,9 +1506,9 @@ Additional Peon configs include:
 |`druid.indexer.task.defaultHadoopCoordinates`|Hadoop version to use with HadoopIndexTasks that do not request a particular version.|`org.apache.hadoop:hadoop-client-api:3.3.6`, `org.apache.hadoop:hadoop-client-runtime:3.3.6`|
 |`druid.indexer.task.defaultRowFlushBoundary`|Highest row count before persisting to disk. Used for indexing generating tasks.|75000|
 |`druid.indexer.task.directoryLockTimeout`|Wait this long for zombie Peons to exit before giving up on their replacements.|PT10M|
-|`druid.indexer.task.gracefulShutdownTimeout`|Wait this long on MiddleManager restart for restorable tasks to gracefully exit.|PT5M|
+|`druid.indexer.task.gracefulShutdownTimeout`|Wait this long on Middle Manager restart for restorable tasks to gracefully exit.|PT5M|
 |`druid.indexer.task.hadoopWorkingPath`|Temporary working directory for Hadoop tasks.|`/tmp/druid-indexing`|
-|`druid.indexer.task.restoreTasksOnRestart`|If true, MiddleManagers will attempt to stop tasks gracefully on shutdown and restore them on restart.|false|
+|`druid.indexer.task.restoreTasksOnRestart`|If true, Middle Managers will attempt to stop tasks gracefully on shutdown and restore them on restart.|false|
 |`druid.indexer.task.ignoreTimestampSpecForDruidInputSource`|If true, tasks using the [Druid input source](../ingestion/input-sources.md) will ignore the provided timestampSpec, and will use the `__time` column of the input datasource. This option is provided for compatibility with ingestion specs written before Druid 0.22.0.|false|
 |`druid.indexer.task.storeEmptyColumns`|Boolean value for whether or not to store empty columns during ingestion. When set to true, Druid stores every column specified in the [`dimensionsSpec`](../ingestion/ingestion-spec.md#dimensionsspec). If you use the string-based schemaless ingestion and don't specify any dimensions to ingest, you must also set [`includeAllDimensions`](../ingestion/ingestion-spec.md#dimensionsspec) for Druid to store empty columns.<br/><br/>If you set `storeEmptyColumns` to false, Druid SQL queries referencing empty columns will fail. If you intend to leave `storeEmptyColumns` disabled, you should either ingest placeholder data for empty columns or else not query on empty columns.<br/><br/>You can overwrite this configuration  by setting `storeEmptyColumns` in the [task context](../ingestion/tasks.md#context-parameters).|true|
 |`druid.indexer.task.tmpStorageBytesPerTask`|Maximum number of bytes per task to be used to store temporary files on disk. This config is generally intended for internal usage. Attempts to set it are very likely to be overwritten by the TaskRunner that executes the task, so be sure of what you expect to happen before directly adjusting this configuration parameter. The config is documented here primarily to provide an understanding of what it means if/when someone sees that it has been set. A value of -1 disables this limit.  |-1|
@@ -2007,9 +2007,9 @@ See [cache configuration](#cache-configuration) for how to configure cache setti
 
 ## Cache configuration
 
-This section describes caching configuration that is common to Broker, Historical, and MiddleManager/Peon processes.
+This section describes caching configuration that is common to Broker, Historical, and Middle Manager/Peon processes.
 
-Caching could optionally be enabled on the Broker, Historical, and MiddleManager/Peon processes. See
+Caching could optionally be enabled on the Broker, Historical, and Middle Manager/Peon processes. See
 [Broker](#broker-caching), [Historical](#historical-caching), and [Peon](#peon-caching) configuration options for how to
 enable it for different processes.
 
@@ -2112,7 +2112,7 @@ If there is an L1 miss and L2 hit, it will also populate L1.
 
 ## General query configuration
 
-This section describes configurations that control behavior of Druid's query types, applicable to Broker, Historical, and MiddleManager processes.
+This section describes configurations that control behavior of Druid's query types, applicable to Broker, Historical, and Middle Manager processes.
 
 ### Overriding default query context values
 
@@ -2163,7 +2163,7 @@ context). If query does have `maxQueuedBytes` in the context, then that value is
 
 ### GroupBy query config
 
-This section describes the configurations for groupBy queries. You can set the runtime properties in the `runtime.properties` file on Broker, Historical, and MiddleManager processes. You can set the query context parameters through the [query context](../querying/query-context.md).
+This section describes the configurations for groupBy queries. You can set the runtime properties in the `runtime.properties` file on Broker, Historical, and Middle Manager processes. You can set the query context parameters through the [query context](../querying/query-context.md).
 
 Supported runtime properties:
 
diff --git a/docs/design/architecture.md b/docs/design/architecture.md
index 188a0b44164..8887308acd0 100644
--- a/docs/design/architecture.md
+++ b/docs/design/architecture.md
@@ -40,8 +40,8 @@ Druid has several types of services:
 * [Broker](../design/broker.md) handles queries from external clients.
 * [Router](../design/router.md) routes requests to Brokers, Coordinators, and Overlords.
 * [Historical](../design/historical.md) stores queryable data.
-* [MiddleManager](../design/middlemanager.md) and [Peon](../design/peons.md) ingest data.
-* [Indexer](../design/indexer.md) serves an alternative to the MiddleManager + Peon task execution system.
+* [Middle Manager](../design/middlemanager.md) and [Peon](../design/peons.md) ingest data.
+* [Indexer](../design/indexer.md) serves an alternative to the Middle Manager + Peon task execution system.
 
 You can view services in the **Services** tab in the web console: 
 
@@ -63,7 +63,7 @@ Master servers divide operations between Coordinator and Overlord services.
 
 #### Overlord service
 
-[Overlord](../design/overlord.md) services watch over the MiddleManager services on the Data servers and are the controllers of data ingestion into Druid. They are responsible for assigning ingestion tasks to MiddleManagers and for coordinating segment publishing.
+[Overlord](../design/overlord.md) services watch over the Middle Manager services on the Data servers and are the controllers of data ingestion into Druid. They are responsible for assigning ingestion tasks to Middle Managers and for coordinating segment publishing.
 
 ### Query server
 
@@ -73,7 +73,7 @@ Query servers divide operations between Broker and Router services.
 
 #### Broker service
 
-[Broker](../design/broker.md) services receive queries from external clients and forward those queries to Data servers. When Brokers receive results from those subqueries, they merge those results and return them to the caller. Typically, you query Brokers rather than querying Historical or MiddleManager services on Data servers directly.
+[Broker](../design/broker.md) services receive queries from external clients and forward those queries to Data servers. When Brokers receive results from those subqueries, they merge those results and return them to the caller. Typically, you query Brokers rather than querying Historical or Middle Manager services on Data servers directly.
 
 #### Router service
 
@@ -85,30 +85,30 @@ The Router service also runs the [web console](../operations/web-console.md), a
 
 A Data server executes ingestion jobs and stores queryable data.
 
-Data servers divide operations between Historical and MiddleManager services.
+Data servers divide operations between Historical and Middle Manager services.
 
 #### Historical service
 
 [**Historical**](../design/historical.md) services handle storage and querying on historical data, including any streaming data that has been in the system long enough to be committed. Historical services download segments from deep storage and respond to queries about these segments. They don't accept writes.
 
-#### MiddleManager service
+#### Middle Manager service
 
-[**MiddleManager**](../design/middlemanager.md) services handle ingestion of new data into the cluster. They are responsible
+[**Middle Manager**](../design/middlemanager.md) services handle ingestion of new data into the cluster. They are responsible
 for reading from external data sources and publishing new Druid segments.
 
 ##### Peon service
 
-[**Peon**](../design/peons.md) services are task execution engines spawned by MiddleManagers. Each Peon runs a separate JVM and is responsible for executing a single task. Peons always run on the same host as the MiddleManager that spawned them.
+[**Peon**](../design/peons.md) services are task execution engines spawned by Middle Managers. Each Peon runs a separate JVM and is responsible for executing a single task. Peons always run on the same host as the Middle Manager that spawned them.
 
 #### Indexer service (optional)
 
-[**Indexer**](../design/indexer.md) services are an alternative to MiddleManagers and Peons. Instead of
+[**Indexer**](../design/indexer.md) services are an alternative to Middle Managers and Peons. Instead of
 forking separate JVM processes per-task, the Indexer runs tasks as individual threads within a single JVM process.
 
-The Indexer is designed to be easier to configure and deploy compared to the MiddleManager + Peon system and to better enable resource sharing across tasks. The Indexer is a newer feature and is currently designated [experimental](../development/experimental.md) due to the fact that its memory management system is still under
+The Indexer is designed to be easier to configure and deploy compared to the Middle Manager + Peon system and to better enable resource sharing across tasks. The Indexer is a newer feature and is currently designated [experimental](../development/experimental.md) due to the fact that its memory management system is still under
 development. It will continue to mature in future versions of Druid.
 
-Typically, you would deploy either MiddleManagers or Indexers, but not both.
+Typically, you would deploy either Middle Managers or Indexers, but not both.
 
 ## Colocation of services
 
@@ -126,11 +126,11 @@ In clusters with very high segment counts, it can make sense to separate the Coo
 You can run the Coordinator and Overlord services as a single combined service by setting the `druid.coordinator.asOverlord.enabled` property.
 For more information, see [Coordinator Operation](../configuration/index.md#coordinator-operation).
 
-### Historicals and MiddleManagers
+### Historicals and Middle Managers
 
-With higher levels of ingestion or query load, it can make sense to deploy the Historical and MiddleManager services on separate hosts to to avoid CPU and memory contention.
+With higher levels of ingestion or query load, it can make sense to deploy the Historical and Middle Manager services on separate hosts to to avoid CPU and memory contention.
 
-The Historical service also benefits from having free memory for memory mapped segments, which can be another reason to deploy the Historical and MiddleManager services separately.
+The Historical service also benefits from having free memory for memory mapped segments, which can be another reason to deploy the Historical and Middle Manager services separately.
 
 ## External dependencies
 
diff --git a/docs/design/indexer.md b/docs/design/indexer.md
index ae9254b9cc2..9d606d1c9a4 100644
--- a/docs/design/indexer.md
+++ b/docs/design/indexer.md
@@ -28,9 +28,9 @@ sidebar_label: "Indexer"
  Its memory management system is still under development and will be significantly enhanced in later releases.
 :::
 
-The Apache Druid Indexer service is an alternative to the MiddleManager + Peon task execution system. Instead of forking a separate JVM process per-task, the Indexer runs tasks as separate threads within a single JVM process.
+The Apache Druid Indexer service is an alternative to the Middle Manager + Peon task execution system. Instead of forking a separate JVM process per-task, the Indexer runs tasks as separate threads within a single JVM process.
 
-The Indexer is designed to be easier to configure and deploy compared to the MiddleManager + Peon system and to better enable resource sharing across tasks.
+The Indexer is designed to be easier to configure and deploy compared to the Middle Manager + Peon system and to better enable resource sharing across tasks.
 
 ## Configuration
 
@@ -38,7 +38,7 @@ For Apache Druid Indexer service configuration, see [Indexer Configuration](../c
 
 ## HTTP endpoints
 
-The Indexer service shares the same HTTP endpoints as the [MiddleManager](../api-reference/service-status-api.md#middlemanager).
+The Indexer service shares the same HTTP endpoints as the [Middle Manager](../api-reference/service-status-api.md#middle-manager).
 
 ## Running
 
@@ -73,7 +73,7 @@ This global limit is evenly divided across the number of task slots configured b
 
 To apply the per-task heap limit, the Indexer overrides `maxBytesInMemory` in task tuning configurations, that is ignoring the default value or any user configured value. It also overrides `maxRowsInMemory` to an essentially unlimited value: the Indexer does not support row limits.
 
-By default, `druid.worker.globalIngestionHeapLimitBytes` is set to 1/6th of the available JVM heap. This default is chosen to align with the default value of `maxBytesInMemory` in task tuning configs when using the MiddleManager + Peon system, which is also 1/6th of the JVM heap.
+By default, `druid.worker.globalIngestionHeapLimitBytes` is set to 1/6th of the available JVM heap. This default is chosen to align with the default value of `maxBytesInMemory` in task tuning configs when using the Middle Manager + Peon system, which is also 1/6th of the JVM heap.
 
 The peak usage for rows held in heap memory relates to the interaction between the `maxBytesInMemory` and `maxPendingPersists` properties in the task tuning configs. When the amount of row data held in-heap by a task reaches the limit specified by `maxBytesInMemory`, a task will persist the in-heap row data. After the persist has been started, the task can again ingest up to `maxBytesInMemory` bytes worth of row data while the persist is running.
 
diff --git a/docs/design/indexing-service.md b/docs/design/indexing-service.md
index 537faa9a405..d7dde33ecdf 100644
--- a/docs/design/indexing-service.md
+++ b/docs/design/indexing-service.md
@@ -27,8 +27,8 @@ The Apache Druid indexing service is a highly-available, distributed service tha
 
 Indexing [tasks](../ingestion/tasks.md) are responsible for creating and [killing](../ingestion/tasks.md#kill) Druid [segments](../design/segments.md).
 
-The indexing service is composed of three main components: [Peons](../design/peons.md) that can run a single task, [MiddleManagers](../design/middlemanager.md) that manage Peons, and an [Overlord](../design/overlord.md) that manages task distribution to MiddleManagers.
-Overlords and MiddleManagers may run on the same process or across multiple processes, while MiddleManagers and Peons always run on the same process.
+The indexing service is composed of three main components: [Peons](../design/peons.md) that can run a single task, [Middle Managers](../design/middlemanager.md) that manage Peons, and an [Overlord](../design/overlord.md) that manages task distribution to Middle Managers.
+Overlords and Middle Managers may run on the same process or across multiple processes, while Middle Managers and Peons always run on the same process.
 
 Tasks are managed using API endpoints on the Overlord service. Please see [Tasks API](../api-reference/tasks-api.md) for more information.
 
diff --git a/docs/design/metadata-storage.md b/docs/design/metadata-storage.md
index 4f66115916d..8071753f3e0 100644
--- a/docs/design/metadata-storage.md
+++ b/docs/design/metadata-storage.md
@@ -149,7 +149,7 @@ parameters across the cluster at runtime.
 
 ### Task-related tables
 
-Task-related tables are created and used by the [Overlord](../design/overlord.md) and [MiddleManager](../design/middlemanager.md) when managing tasks.
+Task-related tables are created and used by the [Overlord](../design/overlord.md) and [Middle Manager](../design/middlemanager.md) when managing tasks.
 
 ### Audit table
 
diff --git a/docs/design/middlemanager.md b/docs/design/middlemanager.md
index 28738295a05..9037b56a6e7 100644
--- a/docs/design/middlemanager.md
+++ b/docs/design/middlemanager.md
@@ -1,7 +1,7 @@
 ---
 id: middlemanager
-title: "MiddleManager service"
-sidebar_label: "MiddleManager"
+title: "Middle Manager service"
+sidebar_label: "Middle Manager"
 ---
 
 <!--
@@ -23,18 +23,18 @@ sidebar_label: "MiddleManager"
   ~ under the License.
   -->
 
-The MiddleManager service is a worker service that executes submitted tasks. MiddleManagers forward tasks to [Peons](../design/peons.md) that run in separate JVMs.
-Druid uses separate JVMs for tasks to isolate resources and logs. Each Peon is capable of running only one task at a time, wheres a MiddleManager may have multiple Peons.
+The Middle Manager service is a worker service that executes submitted tasks. Middle Managers forward tasks to [Peons](../design/peons.md) that run in separate JVMs.
+Druid uses separate JVMs for tasks to isolate resources and logs. Each Peon is capable of running only one task at a time, whereas a Middle Manager may have multiple Peons.
 
 ## Configuration
 
-For Apache Druid MiddleManager service configuration, see [MiddleManager and Peons](../configuration/index.md#middlemanager-and-peons).
+For Apache Druid Middle Manager service configuration, see [Middle Manager and Peons](../configuration/index.md#middle-manager-and-peon).
 
-For basic tuning guidance for the MiddleManager service, see [Basic cluster tuning](../operations/basic-cluster-tuning.md#middlemanager).
+For basic tuning guidance for the Middle Manager service, see [Basic cluster tuning](../operations/basic-cluster-tuning.md#middle-manager).
 
 ## HTTP endpoints
 
-For a list of API endpoints supported by the MiddleManager, see the [Service status API reference](../api-reference/service-status-api.md#middlemanager).
+For a list of API endpoints supported by the Middle Manager, see the [Service status API reference](../api-reference/service-status-api.md#middle-manager).
 
 ## Running
 
diff --git a/docs/design/overlord.md b/docs/design/overlord.md
index 83be16db789..d8458e750a9 100644
--- a/docs/design/overlord.md
+++ b/docs/design/overlord.md
@@ -25,8 +25,8 @@ sidebar_label: "Overlord"
 
 
 The Overlord service is responsible for accepting tasks, coordinating task distribution, creating locks around tasks, and returning statuses to callers. The Overlord can be configured to run in one of two modes - local or remote (local being default).
-In local mode, the Overlord is also responsible for creating Peons for executing tasks. When running the Overlord in local mode, all MiddleManager and Peon configurations must be provided as well.
-Local mode is typically used for simple workflows. In remote mode, the Overlord and MiddleManager are run in separate services and you can run each on a different server.
+In local mode, the Overlord is also responsible for creating Peons for executing tasks. When running the Overlord in local mode, all Middle Manager and Peon configurations must be provided as well.
+Local mode is typically used for simple workflows. In remote mode, the Overlord and Middle Manager are run in separate services and you can run each on a different server.
 This mode is recommended if you intend to use the indexing service as the single endpoint for all Druid indexing.
 
 ## Configuration
@@ -41,7 +41,7 @@ For a list of API endpoints supported by the Overlord, please see the [Service s
 
 ## Blacklisted workers
 
-If a MiddleManager has task failures above a threshold, the Overlord will blacklist these MiddleManagers. No more than 20% of the MiddleManagers can be blacklisted. Blacklisted MiddleManagers will be periodically whitelisted.
+If a Middle Manager has task failures above a threshold, the Overlord will blacklist these Middle Managers. No more than 20% of the Middle Managers can be blacklisted. Blacklisted Middle Managers will be periodically whitelisted.
 
 The following variables can be used to set the threshold and blacklist timeouts.
 
@@ -54,6 +54,6 @@ druid.indexer.runner.maxPercentageBlacklistWorkers
 
 ## Autoscaling
 
-The autoscaling mechanisms currently in place are tightly coupled with our deployment infrastructure but the framework should be in place for other implementations. We are highly open to new implementations or extensions of the existing mechanisms. In our own deployments, MiddleManager services are Amazon AWS EC2 nodes and they are provisioned to register themselves in a [galaxy](https://github.com/ning/galaxy) environment.
+The autoscaling mechanisms currently in place are tightly coupled with our deployment infrastructure but the framework should be in place for other implementations. We are highly open to new implementations or extensions of the existing mechanisms. In our own deployments, Middle Manager services are Amazon AWS EC2 nodes and they are provisioned to register themselves in a [galaxy](https://github.com/ning/galaxy) environment.
 
-If autoscaling is enabled, new MiddleManagers may be added when a task has been in pending state for too long. MiddleManagers may be terminated if they have not run any tasks for a period of time.
+If autoscaling is enabled, new Middle Managers may be added when a task has been in pending state for too long. Middle Managers may be terminated if they have not run any tasks for a period of time.
diff --git a/docs/design/peons.md b/docs/design/peons.md
index 8c2a73a069a..b31bd8ec1a8 100644
--- a/docs/design/peons.md
+++ b/docs/design/peons.md
@@ -23,22 +23,22 @@ sidebar_label: "Peon"
   ~ under the License.
   -->
 
-The Peon service is a task execution engine spawned by the MiddleManager. Each Peon runs a separate JVM and is responsible for executing a single task. Peons always run on the same host as the MiddleManager that spawned them.
+The Peon service is a task execution engine spawned by the Middle Manager. Each Peon runs a separate JVM and is responsible for executing a single task. Peons always run on the same host as the Middle Manager that spawned them.
 
 ## Configuration
 
 For Apache Druid Peon configuration, see [Peon Query Configuration](../configuration/index.md#peon-query-configuration) and [Additional Peon Configuration](../configuration/index.md#additional-peon-configuration).
 
-For basic tuning guidance for MiddleManager tasks, see [Basic cluster tuning](../operations/basic-cluster-tuning.md#task-configurations).
+For basic tuning guidance for Middle Manager tasks, see [Basic cluster tuning](../operations/basic-cluster-tuning.md#task-configurations).
 
 ## HTTP endpoints
 
-Peons run a single task in a single JVM. The MiddleManager is responsible for creating Peons for running tasks.
+Peons run a single task in a single JVM. The Middle Manager is responsible for creating Peons for running tasks.
 Peons should rarely run on their own.
 
 ## Running
 
-The Peon should seldom run separately from the MiddleManager, except for development purposes.
+The Peon should seldom run separately from the Middle Manager, except for development purposes.
 
 ```
 org.apache.druid.cli.Main internal peon <task_file> <status_file>
diff --git a/docs/design/storage.md b/docs/design/storage.md
index 85914e7ada3..fd9783c9283 100644
--- a/docs/design/storage.md
+++ b/docs/design/storage.md
@@ -28,7 +28,7 @@ Druid stores data in datasources, which are similar to tables in a traditional R
 
 ![Segment timeline](../assets/druid-timeline.png)
 
-A datasource may have anywhere from just a few segments, up to hundreds of thousands and even millions of segments. Each segment is created by a MiddleManager as mutable and uncommitted. Data is queryable as soon as it is added to an uncommitted segment. The segment building process accelerates later queries by producing a data file that is compact and indexed:
+A datasource may have anywhere from just a few segments, up to hundreds of thousands and even millions of segments. Each segment is created by a Middle Manager as mutable and uncommitted. Data is queryable as soon as it is added to an uncommitted segment. The segment building process accelerates later queries by producing a data file that is compact and indexed:
 
 - Conversion to columnar format
 - Indexing with bitmap indexes
@@ -37,7 +37,7 @@ A datasource may have anywhere from just a few segments, up to hundreds of thous
     - Bitmap compression for bitmap indexes
     - Type-aware compression for all columns
 
-Periodically, segments are committed and published to [deep storage](deep-storage.md), become immutable, and move from MiddleManagers to the Historical services. An entry about the segment is also written to the [metadata store](metadata-storage.md). This entry is a self-describing bit of metadata about the segment, including things like the schema of the segment, its size, and its location on deep storage. These entries tell the Coordinator what data is available on the cluster.
+Periodically, segments are committed and published to [deep storage](deep-storage.md), become immutable, and move from Middle Managers to the Historical services. An entry about the segment is also written to the [metadata store](metadata-storage.md). This entry is a self-describing bit of metadata about the segment, including things like the schema of the segment, its size, and its location on deep storage. These entries tell the Coordinator what data is available on the cluster.
 
 For details on the segment file format, see [segment files](segments.md).
 
diff --git a/docs/design/zookeeper.md b/docs/design/zookeeper.md
index eb8bea57741..a01f12ed74d 100644
--- a/docs/design/zookeeper.md
+++ b/docs/design/zookeeper.md
@@ -41,7 +41,7 @@ The operations that happen over ZK are
 1.  [Coordinator](../design/coordinator.md) leader election
 2.  Segment "publishing" protocol from [Historical](../design/historical.md)
 3.  [Overlord](../design/overlord.md) leader election
-4.  [Overlord](../design/overlord.md) and [MiddleManager](../design/middlemanager.md) task management
+4.  [Overlord](../design/overlord.md) and [Middle Manager](../design/middlemanager.md) task management
 
 ## Coordinator Leader Election
 
diff --git a/docs/development/extensions-contrib/gce-extensions.md b/docs/development/extensions-contrib/gce-extensions.md
index 17a69c72f23..2a8da661547 100644
--- a/docs/development/extensions-contrib/gce-extensions.md
+++ b/docs/development/extensions-contrib/gce-extensions.md
@@ -32,7 +32,7 @@ of GCE (MIG from now on). This choice has been made to ease the configuration of
 management.
 
 For this reason, in order to use this extension, the user must have created
-1. An instance template with the right machine type and image to bu used to run the MiddleManager
+1. An instance template with the right machine type and image to bu used to run the Middle Manager
 2. A MIG that has been configured to use the instance template created in the point above
 
 Moreover, in order to be able to rescale the machines in the MIG, the Overlord must run with a service account
@@ -98,6 +98,6 @@ for parameters other than the ones specified here, such as `selectStrategy` etc.
 - The module internally uses the [ListManagedInstances](https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers/listManagedInstances)
  call from the API and, while the documentation of the API states that the call can be paged through using the
  `pageToken` argument, the responses to such call do not provide any `nextPageToken` to set such parameter. This means
- that the extension can operate safely with a maximum of 500 MiddleManagers instances at any time (the maximum number
+ that the extension can operate safely with a maximum of 500 Middle Managers instances at any time (the maximum number
  of instances to be returned for each call).
  
\ No newline at end of file
diff --git a/docs/development/extensions-contrib/prometheus.md b/docs/development/extensions-contrib/prometheus.md
index 37bcbd4e2e2..ddbd9da4643 100644
--- a/docs/development/extensions-contrib/prometheus.md
+++ b/docs/development/extensions-contrib/prometheus.md
@@ -51,7 +51,7 @@ All the configuration parameters for the Prometheus emitter are under `druid.emi
 
 ### Ports for colocated Druid processes
 
-In certain instances, Druid processes may be colocated on the same host. For example, the Broker and Router may share the same server. Other colocated processes include the Historical and MiddleManager or the Coordinator and Overlord. When you have colocated processes, specify `druid.emitter.prometheus.port` separately for each process on each host. For example, even if the Broker and Router share the same host, the Broker runtime properties and the Router runtime properties each need to list `druid.emitter.prometheus.port`, and the port value for both must be different.
+In certain instances, Druid processes may be colocated on the same host. For example, the Broker and Router may share the same server. Other colocated processes include the Historical and Middle Manager or the Coordinator and Overlord. When you have colocated processes, specify `druid.emitter.prometheus.port` separately for each process on each host. For example, even if the Broker and Router share the same host, the Broker runtime properties and the Router runtime properties each need to list `druid.emitter.prometheus.port`, and the port value for both must be different.
 
 ### Override properties for Peon Tasks
 
diff --git a/docs/development/extensions-core/s3.md b/docs/development/extensions-core/s3.md
index 9690c27e4ca..2565451c272 100644
--- a/docs/development/extensions-core/s3.md
+++ b/docs/development/extensions-core/s3.md
@@ -117,7 +117,7 @@ The AWS SDK requires that a target region be specified.  You can set these by us
 For example, to set the region to 'us-east-1' through system properties:
 
 * Add `-Daws.region=us-east-1` to the `jvm.config` file for all Druid services.
-* Add `-Daws.region=us-east-1` to `druid.indexer.runner.javaOpts` in [Middle Manager configuration](../../configuration/index.md#middlemanager-configuration) so that the property will be passed to Peon (worker) processes.
+* Add `-Daws.region=us-east-1` to `druid.indexer.runner.javaOpts` in [Middle Manager configuration](../../configuration/index.md#middle-manager-configuration) so that the property will be passed to Peon (worker) processes.
 
 ### Connecting to S3 configuration
 
diff --git a/docs/ingestion/hadoop.md b/docs/ingestion/hadoop.md
index 96373e27517..f238be3405a 100644
--- a/docs/ingestion/hadoop.md
+++ b/docs/ingestion/hadoop.md
@@ -149,7 +149,7 @@ For example, using the static input paths:
 ```
 
 You can also read from cloud storage such as Amazon S3 or Google Cloud Storage.
-To do so, you need to install the necessary library under Druid's classpath in _all MiddleManager or Indexer processes_.
+To do so, you need to install the necessary library under Druid's classpath in _all Middle Manager or Indexer processes_.
 For S3, you can run the below command to install the [Hadoop AWS module](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/).
 
 ```bash
@@ -157,7 +157,7 @@ java -classpath "${DRUID_HOME}lib/*" org.apache.druid.cli.Main tools pull-deps -
 cp ${DRUID_HOME}/hadoop-dependencies/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar ${DRUID_HOME}/extensions/druid-hdfs-storage/
 ```
 
-Once you install the Hadoop AWS module in all MiddleManager and Indexer processes, you can put
+Once you install the Hadoop AWS module in all Middle Manager and Indexer processes, you can put
 your S3 paths in the inputSpec with the below job properties.
 For more configurations, see the [Hadoop AWS module](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/).
 
@@ -175,8 +175,8 @@ For more configurations, see the [Hadoop AWS module](https://hadoop.apache.org/d
 ```
 
 For Google Cloud Storage, you need to install [GCS connector jar](https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md)
-under `${DRUID_HOME}/hadoop-dependencies` in _all MiddleManager or Indexer processes_.
-Once you install the GCS Connector jar in all MiddleManager and Indexer processes, you can put
+under `${DRUID_HOME}/hadoop-dependencies` in _all Middle Manager or Indexer processes_.
+Once you install the GCS Connector jar in all Middle Manager and Indexer processes, you can put
 your Google Cloud Storage paths in the inputSpec with the below job properties.
 For more configurations, see the [instructions to configure Hadoop](https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md#configure-hadoop),
 [GCS core default](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/v2.0.0/gcs/conf/gcs-core-default.xml)
diff --git a/docs/ingestion/index.md b/docs/ingestion/index.md
index d500b1d9bc2..bc189f4b592 100644
--- a/docs/ingestion/index.md
+++ b/docs/ingestion/index.md
@@ -27,7 +27,7 @@ Loading data in Druid is called _ingestion_ or _indexing_. When you ingest data
 your source system and stores it in data files called [_segments_](../design/segments.md).
 In general, segment files contain a few million rows each.
 
-For most ingestion methods, the Druid [MiddleManager](../design/middlemanager.md) processes or the
+For most ingestion methods, the Druid [Middle Manager](../design/middlemanager.md) processes or the
 [Indexer](../design/indexer.md) processes load your source data. The sole exception is Hadoop-based ingestion, which
 uses a Hadoop MapReduce job on YARN.
 
diff --git a/docs/ingestion/input-sources.md b/docs/ingestion/input-sources.md
index fb8e1f98c91..c6ba2e5b49e 100644
--- a/docs/ingestion/input-sources.md
+++ b/docs/ingestion/input-sources.md
@@ -875,7 +875,7 @@ Each of the SQL queries will be run in its own sub-task and thus for the above e
 
 Compared to the other native batch input sources, SQL input source behaves differently in terms of reading the input data. Therefore, consider the following points before using this input source in a production environment:
 
-* During indexing, each sub-task would execute one of the SQL queries and the results are stored locally on disk. The sub-tasks then proceed to read the data from these local input files and generate segments. Presently, there isn’t any restriction on the size of the generated files and this would require the MiddleManagers or Indexers to have sufficient disk capacity based on the volume of data being indexed.
+* During indexing, each sub-task would execute one of the SQL queries and the results are stored locally on disk. The sub-tasks then proceed to read the data from these local input files and generate segments. Presently, there isn’t any restriction on the size of the generated files and this would require the Middle Managers or Indexers to have sufficient disk capacity based on the volume of data being indexed.
 
 * Filtering the SQL queries based on the intervals specified in the `granularitySpec` can avoid unwanted data being retrieved and stored locally by the indexing sub-tasks. For example, if the `intervals` specified in the `granularitySpec` is `["2013-01-01/2013-01-02"]` and the SQL query is `SELECT * FROM table1`, `SqlInputSource` will read all the data for `table1` based on the query, even though only data between the intervals specified will be indexed into Druid.
 
diff --git a/docs/ingestion/kafka-ingestion.md b/docs/ingestion/kafka-ingestion.md
index 7eb393017d1..5c4dc0856a8 100644
--- a/docs/ingestion/kafka-ingestion.md
+++ b/docs/ingestion/kafka-ingestion.md
@@ -36,7 +36,7 @@ This topic contains configuration information for the Kafka indexing service sup
 
 ## Setup
 
-To use the Kafka indexing service, you must first load the `druid-kafka-indexing-service` extension on both the Overlord and the MiddleManager. See [Loading extensions](../configuration/extensions.md) for more information.
+To use the Kafka indexing service, you must first load the `druid-kafka-indexing-service` extension on both the Overlord and the Middle Manager. See [Loading extensions](../configuration/extensions.md) for more information.
 
 ## Supervisor spec configuration
 
@@ -426,7 +426,7 @@ For configuration properties shared across all streaming ingestion methods, refe
 
 Druid assigns Kafka partitions to each Kafka indexing task. A task writes the events it consumes from Kafka into a single segment for the segment granularity interval until it reaches one of the following limits: `maxRowsPerSegment`, `maxTotalRows`, or `intermediateHandoffPeriod`. At this point, the task creates a new partition for this segment granularity to contain subsequent events.
 
-The Kafka indexing task also does incremental hand-offs. Therefore, segments become available as they are ready and you don't have to wait for all segments until the end of the task duration. When the task reaches one of `maxRowsPerSegment`, `maxTotalRows`, or `intermediateHandoffPeriod`, it hands off all the segments and creates a new set of segments for further events. This allows the task to run for longer durations without accumulating old segments locally on MiddleManager services.
+The Kafka indexing task also does incremental hand-offs. Therefore, segments become available as they are ready and you don't have to wait for all segments until the end of the task duration. When the task reaches one of `maxRowsPerSegment`, `maxTotalRows`, or `intermediateHandoffPeriod`, it hands off all the segments and creates a new set of segments for further events. This allows the task to run for longer durations without accumulating old segments locally on Middle Manager services.
 
 The Kafka indexing service may still produce some small segments. For example, consider the following scenario:
 - Task duration is 4 hours.
diff --git a/docs/ingestion/kinesis-ingestion.md b/docs/ingestion/kinesis-ingestion.md
index 8ea2cfd4af3..2a855b7b7ff 100644
--- a/docs/ingestion/kinesis-ingestion.md
+++ b/docs/ingestion/kinesis-ingestion.md
@@ -33,7 +33,7 @@ This topic contains configuration information for the Kinesis indexing service s
 
 ## Setup
 
-To use the Kinesis indexing service, you must first load the `druid-kinesis-indexing-service` core extension on both the Overlord and the MiddleManager. See [Loading extensions](../configuration/extensions.md#loading-extensions) for more information.
+To use the Kinesis indexing service, you must first load the `druid-kinesis-indexing-service` core extension on both the Overlord and the Middle Manager. See [Loading extensions](../configuration/extensions.md#loading-extensions) for more information.
 
 Review [Known issues](#known-issues) before deploying the `druid-kinesis-indexing-service` extension to production.
 
@@ -249,7 +249,7 @@ At this point, the task creates a new shard for this segment granularity to cont
 
 The Kinesis indexing task also performs incremental hand-offs so that the segments created by the task are not held up until the task duration is over.
 When the task reaches one of the `maxRowsPerSegment`, `maxTotalRows`, or `intermediateHandoffPeriod` limits, it hands off all the segments and creates a new set of segments for further events. This allows the task to run for longer durations
-without accumulating old segments locally on MiddleManager services.
+without accumulating old segments locally on Middle Manager services.
 
 The Kinesis indexing service may still produce some small segments.
 For example, consider the following scenario:
diff --git a/docs/ingestion/native-batch.md b/docs/ingestion/native-batch.md
index 398fea9f69a..be3219a5003 100644
--- a/docs/ingestion/native-batch.md
+++ b/docs/ingestion/native-batch.md
@@ -55,7 +55,7 @@ The parallel task type `index_parallel` is a task for multi-threaded batch index
 The `index_parallel` task is a supervisor task that orchestrates
 the whole indexing process. The supervisor task splits the input data and creates worker tasks to process the individual portions of data.
 
-Druid issues the worker tasks to the Overlord. The Overlord schedules and runs the workers on MiddleManagers or Indexers. After a worker task successfully processes the assigned input portion, it reports the resulting segment list to the Supervisor task.
+Druid issues the worker tasks to the Overlord. The Overlord schedules and runs the workers on Middle Managers or Indexers. After a worker task successfully processes the assigned input portion, it reports the resulting segment list to the Supervisor task.
 
 The Supervisor task periodically checks the status of worker tasks. If a task fails, the Supervisor retries the task until the number of retries reaches the configured limit. If all worker tasks succeed, it publishes the reported segments at once and finalizes ingestion.
 
@@ -369,11 +369,11 @@ the Parallel task splits the input data based on the split hint spec
 and assigns each split to a worker task. Each worker task (type `partial_index_generate`) reads the assigned split, and partitions rows by the time chunk from `segmentGranularity` (primary partition key) in the `granularitySpec`
 and then by the hash value of `partitionDimensions` (secondary partition key) in the `partitionsSpec`.
 The partitioned data is stored in local storage of
-the [middleManager](../design/middlemanager.md) or the [indexer](../design/indexer.md).
+the [middle Manager](../design/middlemanager.md) or the [indexer](../design/indexer.md).
 
 The `partial segment merge` phase is similar to the Reduce phase in MapReduce.
 The Parallel task spawns a new set of worker tasks (type `partial_index_generic_merge`) to merge the partitioned data created in the previous phase. Here, the partitioned data is shuffled based on
-the time chunk and the hash value of `partitionDimensions` to be merged; each worker task reads the data falling in the same time chunk and the same hash value from multiple MiddleManager/Indexer processes and merges them to create the final segments. Finally, they push the final segments to the deep storage at once.
+the time chunk and the hash value of `partitionDimensions` to be merged; each worker task reads the data falling in the same time chunk and the same hash value from multiple Middle Manager/Indexer processes and merges them to create the final segments. Finally, they push the final segments to the deep storage at once.
 
 ##### Hash partition function
 
@@ -426,12 +426,12 @@ to create partitioned data. Each worker task reads a split created as in the pre
 partitions rows by the time chunk from the `segmentGranularity` (primary partition key) in the `granularitySpec`
 and then by the range partitioning found in the previous phase.
 The partitioned data is stored in local storage of
-the [middleManager](../design/middlemanager.md) or the [indexer](../design/indexer.md).
+the [Middle Manager](../design/middlemanager.md) or the [indexer](../design/indexer.md).
 
 In the `partial segment merge` phase, the parallel index task spawns a new set of worker tasks (type `partial_index_generic_merge`) to merge the partitioned
 data created in the previous phase. Here, the partitioned data is shuffled based on
 the time chunk and the value of `partitionDimension`; each worker task reads the segments
-falling in the same partition of the same range from multiple MiddleManager/Indexer processes and merges
+falling in the same partition of the same range from multiple Middle Manager/Indexer processes and merges
 them to create the final segments. Finally, they push the final segments to the deep storage.
 
 :::info
diff --git a/docs/ingestion/streaming.md b/docs/ingestion/streaming.md
index f0f777b6c62..942ed46b982 100644
--- a/docs/ingestion/streaming.md
+++ b/docs/ingestion/streaming.md
@@ -28,7 +28,7 @@ Apache Druid can consume data streams from the following external streaming sour
 * Amazon Kinesis through the bundled [Kinesis indexing service](kinesis-ingestion.md) extension.
 
 Each indexing service provides real-time data ingestion with exactly-once stream processing guarantee.
-To use either of the streaming ingestion methods, you must first load the associated extension on both the Overlord and the MiddleManager. See [Loading extensions](../configuration/extensions.md#loading-extensions) for more information.
+To use either of the streaming ingestion methods, you must first load the associated extension on both the Overlord and the Middle Manager. See [Loading extensions](../configuration/extensions.md#loading-extensions) for more information.
 
 Streaming ingestion is controlled by a continuously running [supervisor](supervisor.md).
 The supervisor oversees the state of indexing tasks to coordinate handoffs, manage failures, and ensure that scalability and replication requirements are maintained.
diff --git a/docs/ingestion/supervisor.md b/docs/ingestion/supervisor.md
index 70939adb633..d5293ae581f 100644
--- a/docs/ingestion/supervisor.md
+++ b/docs/ingestion/supervisor.md
@@ -393,7 +393,7 @@ For information on how to terminate a supervisor by API, see [Supervisors: Termi
 
 ## Capacity planning
 
-Indexing tasks run on MiddleManagers and are limited by the resources available in the MiddleManager cluster. In particular, you should make sure that you have sufficient worker capacity, configured using the
+Indexing tasks run on Middle Managers and are limited by the resources available in the Middle Manager cluster. In particular, you should make sure that you have sufficient worker capacity, configured using the
 `druid.worker.capacity` property, to handle the configuration in the supervisor spec. Note that worker capacity is
 shared across all types of indexing tasks, so you should plan your worker capacity to handle your total indexing load, such as batch processing, streaming tasks, and merging tasks. If your workers run out of capacity, indexing tasks queue and wait for the next available worker. This may cause queries to return partial results but will not result in data loss, assuming the tasks run before the stream purges those sequence numbers.
 
diff --git a/docs/ingestion/tasks.md b/docs/ingestion/tasks.md
index 743a46cc854..ba93e9c1f00 100644
--- a/docs/ingestion/tasks.md
+++ b/docs/ingestion/tasks.md
@@ -458,7 +458,7 @@ To enable batched segment allocation on the overlord, set  `druid.indexer.tasklo
 The task context is used for various individual task configuration.
 Specify task context configurations in the `context` field of the ingestion spec.
 When configuring [automatic compaction](../data-management/automatic-compaction.md), set the task context configurations in `taskContext` rather than in `context`.
-The settings get passed into the `context` field of the compaction tasks issued to MiddleManagers.
+The settings get passed into the `context` field of the compaction tasks issued to Middle Managers.
 
 The following parameters apply to all task types.
 
@@ -477,18 +477,18 @@ Logs are created by ingestion tasks as they run. You can configure Druid to push
 
 Once the task has been submitted to the Overlord it remains `WAITING` for locks to be acquired. Worker slot allocation is then `PENDING` until the task can actually start executing.
 
-The task then starts creating logs in a local directory of the middle manager (or indexer) in a `log` directory for the specific `taskId` at [`druid.worker.baseTaskDirs`](../configuration/index.md#middlemanager-configuration).
+The task then starts creating logs in a local directory of the middle manager (or indexer) in a `log` directory for the specific `taskId` at [`druid.worker.baseTaskDirs`](../configuration/index.md#middle-manager-configuration).
 
 When the task completes - whether it succeeds or fails - the middle manager (or indexer) will push the task log file into the location specified in [`druid.indexer.logs`](../configuration/index.md#task-logging).
 
-Task logs on the Druid web console are retrieved via an [API](../api-reference/service-status-api.md#overlord) on the Overlord. It automatically detects where the log file is, either in the middleManager / indexer or in long-term storage, and passes it back.
+Task logs on the Druid web console are retrieved via an [API](../api-reference/service-status-api.md#overlord) on the Overlord. It automatically detects where the log file is, either in the Middle Manager / indexer or in long-term storage, and passes it back.
 
 If you don't see the log file in long-term storage, it means either:
 
-1. the middleManager / indexer failed to push the log file to deep storage or
-2. the task did not complete.
+- the Middle Manager / indexer failed to push the log file to deep storage or
+- the task did not complete.
 
-You can check the middleManager / indexer logs locally to see if there was a push failure. If there was not, check the Overlord's own process logs to see why the task failed before it started.
+You can check the Middle Manager / indexer logs locally to see if there was a push failure. If there was not, check the Overlord's own process logs to see why the task failed before it started.
 
 :::info
  If you are running the indexing service in remote mode, the task logs must be stored in S3, Azure Blob Store, Google Cloud Storage or HDFS.
diff --git a/docs/operations/basic-cluster-tuning.md b/docs/operations/basic-cluster-tuning.md
index 7224276da37..32db4863e66 100644
--- a/docs/operations/basic-cluster-tuning.md
+++ b/docs/operations/basic-cluster-tuning.md
@@ -176,29 +176,29 @@ To estimate total memory usage of the Broker under these guidelines:
 - Heap: allocated heap size
 - Direct Memory: `(druid.processing.numMergeBuffers + 1) * druid.processing.buffer.sizeBytes`
 
-### MiddleManager
+### Middle Manager
 
-The MiddleManager is a lightweight task controller/manager that launches Task processes, which perform ingestion work.
+The Middle Manager is a lightweight task controller/manager that launches Task processes, which perform ingestion work.
 
-#### MiddleManager heap sizing
+#### Middle Manager heap sizing
 
-The MiddleManager itself does not require much resources, you can set the heap to ~128MiB generally.
+The Middle Manager itself does not require much resources, you can set the heap to ~128MiB generally.
 
 #### SSD storage
 
-We recommend using SSDs for storage on the MiddleManagers, as the Tasks launched by MiddleManagers handle segment data stored on disk.
+We recommend using SSDs for storage on the Middle Managers, as the Tasks launched by Middle Managers handle segment data stored on disk.
 
 #### Task Count
 
-The number of tasks a MiddleManager can launch is controlled by the `druid.worker.capacity` setting.
+The number of tasks a Middle Manager can launch is controlled by the `druid.worker.capacity` setting.
 
 The number of workers needed in your cluster depends on how many concurrent ingestion tasks you need to run for your use cases. The number of workers that can be launched on a given machine depends on the size of resources allocated per worker and available system resources.
 
-You can allocate more MiddleManager machines to your cluster to add task capacity.
+You can allocate more Middle Manager machines to your cluster to add task capacity.
 
 #### Task configurations
 
-The following section below describes configuration for Tasks launched by the MiddleManager. The Tasks can be queried and perform ingestion workloads, so they require more resources than the MM.
+The following section below describes configuration for Tasks launched by the Middle Manager. The Tasks can be queried and perform ingestion workloads, so they require more resources than the MM.
 
 ##### Task heap sizing
 
@@ -249,7 +249,7 @@ To estimate total memory usage of a Task under these guidelines:
 - Heap: `1GiB + (2 * total size of lookup maps)`
 - Direct Memory: `(druid.processing.numThreads + druid.processing.numMergeBuffers + 1) * druid.processing.buffer.sizeBytes`
 
-The total memory usage of the MiddleManager + Tasks:
+The total memory usage of the Middle Manager + Tasks:
 
 `MM heap size + druid.worker.capacity * (single task memory usage)`
 
@@ -435,7 +435,7 @@ Additionally, for large JVM heaps, here are a few Garbage Collection efficiency
 
 
 - Mount /tmp on tmpfs. See [The Four Month Bug: JVM statistics cause garbage collection pauses](http://www.evanjones.ca/jvm-mmap-pause.html).
-- On Disk-IO intensive processes (e.g., Historical and MiddleManager), GC and Druid logs should be written to a different disk than where data is written.
+- On Disk-IO intensive processes (e.g., Historical and Middle Manager), GC and Druid logs should be written to a different disk than where data is written.
 - Disable [Transparent Huge Pages](https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html).
 - Try disabling biased locking by using `-XX:-UseBiasedLocking` JVM flag. See [Logging Stop-the-world Pauses in JVM](https://dzone.com/articles/logging-stop-world-pauses-jvm).
 
@@ -447,7 +447,7 @@ We recommend using UTC timezone for all your events and across your hosts, not j
 
 #### SSDs
 
-SSDs are highly recommended for Historical, MiddleManager, and Indexer processes if you are not running a cluster that is entirely in memory. SSDs can greatly mitigate the time required to page data in and out of memory.
+SSDs are highly recommended for Historical, Middle Manager, and Indexer processes if you are not running a cluster that is entirely in memory. SSDs can greatly mitigate the time required to page data in and out of memory.
 
 #### JBOD vs RAID
 
@@ -455,11 +455,11 @@ Historical processes store large number of segments on Disk and support specifyi
 
 #### Swap space
 
-We recommend _not_ using swap space for Historical, MiddleManager, and Indexer processes since due to the large number of memory mapped segment files can lead to poor and unpredictable performance.
+We recommend _not_ using swap space for Historical, Middle Manager, and Indexer processes since due to the large number of memory mapped segment files can lead to poor and unpredictable performance.
 
 #### Linux limits
 
-For Historical, MiddleManager, and Indexer processes (and for really large clusters, Broker processes), you might need to adjust some Linux system limits to account for a large number of open files, a large number of network connections, or a large number of memory mapped files.
+For Historical, Middle Manager, and Indexer processes (and for really large clusters, Broker processes), you might need to adjust some Linux system limits to account for a large number of open files, a large number of network connections, or a large number of memory mapped files.
 
 ##### ulimit
 
@@ -467,4 +467,4 @@ The limit on the number of open files can be set permanently by editing `/etc/se
 
 ##### max_map_count
 
-Historical processes and to a lesser extent, MiddleManager and Indexer processes memory map segment files, so depending on the number of segments per server, `/proc/sys/vm/max_map_count` might also need to be adjusted. Depending on the variant of Linux, this might be done via `sysctl` by placing a file in `/etc/sysctl.d/` that sets `vm.max_map_count`.
+Historical processes and to a lesser extent, Middle Manager and Indexer processes memory map segment files, so depending on the number of segments per server, `/proc/sys/vm/max_map_count` might also need to be adjusted. Depending on the variant of Linux, this might be done via `sysctl` by placing a file in `/etc/sysctl.d/` that sets `vm.max_map_count`.
diff --git a/docs/operations/other-hadoop.md b/docs/operations/other-hadoop.md
index f5e5839a907..ba19a832643 100644
--- a/docs/operations/other-hadoop.md
+++ b/docs/operations/other-hadoop.md
@@ -55,7 +55,7 @@ Generally, you should only set one of these parameters, not both.
 These properties can be set in either one of the following ways:
 
 - Using the task definition, e.g. add `"mapreduce.job.classloader": "true"` to the `jobProperties` of the `tuningConfig` of your indexing task (see the [Hadoop batch ingestion documentation](../ingestion/hadoop.md)).
-- Using system properties, e.g. on the MiddleManager set `druid.indexer.runner.javaOpts=... -Dhadoop.mapreduce.job.classloader=true` in [Middle Manager configuration](../configuration/index.md#middlemanager-configuration).
+- Using system properties, e.g. on the Middle Manager set `druid.indexer.runner.javaOpts=... -Dhadoop.mapreduce.job.classloader=true` in [Middle Manager configuration](../configuration/index.md#middle-manager-configuration).
 
 ### Overriding specific classes
 
diff --git a/docs/operations/rolling-updates.md b/docs/operations/rolling-updates.md
index b9471bc6b8e..07cf3b4c8fc 100644
--- a/docs/operations/rolling-updates.md
+++ b/docs/operations/rolling-updates.md
@@ -30,7 +30,7 @@ following order:
 2. Middle Manager and Indexer (if any)
 3. Broker
 4. Router
-5. Overlord (Note that you can upgrade the Overlord before any MiddleManager processes if you use [autoscaling-based replacement](#autoscaling-based-replacement).)
+5. Overlord (Note that you can upgrade the Overlord before any Middle Manager processes if you use [autoscaling-based replacement](#autoscaling-based-replacement).)
 6. Coordinator ( or merged Coordinator+Overlord )
 
 If you need to do a rolling downgrade, reverse the order and start with the Coordinator processes.
@@ -70,14 +70,14 @@ Middle Managers can be gracefully terminated using the "disable" API. This works
 even tasks that are not restorable.
 
 To prepare a Middle Manager for update, send a POST request to
-`<MiddleManager_IP:PORT>/druid/worker/v1/disable`. The Overlord will now no longer send tasks to
+`<Middle_Manager_IP:PORT>/druid/worker/v1/disable`. The Overlord will now no longer send tasks to
 this Middle Manager. Tasks that have already started will run to completion. Current state can be checked
-using `<MiddleManager_IP:PORT>/druid/worker/v1/enabled` .
+using `<Middle_Manager_IP:PORT>/druid/worker/v1/enabled` .
 
-To view all existing tasks, send a GET request to `<MiddleManager_IP:PORT>/druid/worker/v1/tasks`.
+To view all existing tasks, send a GET request to `<Middle_Manager_IP:PORT>/druid/worker/v1/tasks`.
 When this list is empty, you can safely update the Middle Manager. After the Middle Manager starts
 back up, it is automatically enabled again. You can also manually enable Middle Managers by POSTing
-to `<MiddleManager_IP:PORT>/druid/worker/v1/enable`.
+to `<Middle_Manager_IP:PORT>/druid/worker/v1/enable`.
 
 ### Autoscaling-based replacement
 
diff --git a/docs/querying/groupbyquery.md b/docs/querying/groupbyquery.md
index a11f82d124a..2b6be9904e6 100644
--- a/docs/querying/groupbyquery.md
+++ b/docs/querying/groupbyquery.md
@@ -337,7 +337,7 @@ dictionary that can spill to disk. The outer query is run on the Broker in a sin
 
 ### Configurations
 
-This section describes the configurations for groupBy queries. You can set the runtime properties in the `runtime.properties` file on Broker, Historical, and MiddleManager processes. You can set the query context parameters through the [query context](query-context.md).
+This section describes the configurations for groupBy queries. You can set the runtime properties in the `runtime.properties` file on Broker, Historical, and Middle Manager processes. You can set the query context parameters through the [query context](query-context.md).
 
 Supported runtime properties:
 
diff --git a/docs/querying/query-execution.md b/docs/querying/query-execution.md
index 21d6f5256f0..123b80738c4 100644
--- a/docs/querying/query-execution.md
+++ b/docs/querying/query-execution.md
@@ -47,7 +47,7 @@ by range using the [`single_dim` partitionsSpec](../ingestion/native-batch.md#pa
 the dimension used for partitioning.
 
 3. The Broker, having pruned the list of segments for the query, forwards the query to data servers (like Historicals
-and tasks running on MiddleManagers) that are currently serving those segments.
+and tasks running on Middle Managers) that are currently serving those segments.
 
 4. For all query types except [Scan](scan-query.md), data servers process each segment in parallel and generate partial
 results for each segment. The specific processing that is done depends on the query type. These partial results may be
diff --git a/docs/querying/query-processing.md b/docs/querying/query-processing.md
index c94a4bf9cec..b4ecd006f07 100644
--- a/docs/querying/query-processing.md
+++ b/docs/querying/query-processing.md
@@ -27,8 +27,8 @@ This topic provides a high-level overview of how Apache Druid distributes and pr
 The general flow is as follows:
 
 1. A query enters the [Broker](../design/broker.md) service, which identifies the segments with data that may pertain to that query. The list of segments is always pruned by time, and may also be pruned by other attributes depending on how the datasource is partitioned.
-2. The Broker identifies which [Historical](../design/historical.md) and [MiddleManager](../design/middlemanager.md) services are serving those segments and distributes a rewritten subquery to each of the services.
-3. The Historical and MiddleManager services execute each subquery and return results to the Broker.
+2. The Broker identifies which [Historical](../design/historical.md) and [Middle Manager](../design/middlemanager.md) services are serving those segments and distributes a rewritten subquery to each of the services.
+3. The Historical and Middle Manager services execute each subquery and return results to the Broker.
 4. The Broker merges the partial results to get the final answer, which it returns to the original caller.
 
 Druid uses time and attribute pruning to minimize the data it must scan for each query.
diff --git a/docs/release-info/upgrade-notes.md b/docs/release-info/upgrade-notes.md
index 52adccabbc7..d3c6ce1cba3 100644
--- a/docs/release-info/upgrade-notes.md
+++ b/docs/release-info/upgrade-notes.md
@@ -74,7 +74,7 @@ The following are the changes to the default values for the Coordinator service:
 
 #### `GoogleTaskLogs` upload buffer size
 
-Changed the upload buffer size in `GoogleTaskLogs` to 1 MB instead of 15 MB to allow more uploads in parallel and prevent the MiddleManager service from running out of memory.
+Changed the upload buffer size in `GoogleTaskLogs` to 1 MB instead of 15 MB to allow more uploads in parallel and prevent the Middle Manager service from running out of memory.
 
 [#16236](https://github.com/apache/druid/pull/16236)
 
diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md
index f1bea59ff95..8c4ad3e1234 100644
--- a/docs/tutorials/cluster.md
+++ b/docs/tutorials/cluster.md
@@ -32,7 +32,7 @@ your needs.
 This simple cluster will feature:
 
  - A Master server to host the Coordinator and Overlord processes
- - Two scalable, fault-tolerant Data servers running Historical and MiddleManager processes
+ - Two scalable, fault-tolerant Data servers running Historical and Middle Manager processes
  - A query server, hosting the Druid Broker and Router processes
 
 In production, we recommend deploying multiple Master servers and multiple Query servers in a fault-tolerant configuration based on your specific fault-tolerance needs, but you can get started quickly with one Master and one Query server and add more servers later.
@@ -58,7 +58,7 @@ Example Master server configurations that have been sized for this hardware can
 
 #### Data server
 
-Historicals and MiddleManagers can be colocated on the same server to handle the actual data in your cluster. These servers benefit greatly from CPU, RAM,
+Historicals and Middle Managers can be colocated on the same server to handle the actual data in your cluster. These servers benefit greatly from CPU, RAM,
 and SSDs.
 
 In this example, we will be deploying the equivalent of two AWS [i3.4xlarge](https://aws.amazon.com/ec2/instance-types/i3/) instances.
@@ -117,7 +117,7 @@ In a clustered deployment, having multiple Data servers is a good idea for fault
 
 When choosing the Data server hardware, you can choose a split factor `N`, divide the original CPU/RAM of the single-server deployment by `N`, and deploy `N` Data servers of reduced size in the new cluster.
 
-Instructions for adjusting the Historical/MiddleManager configs for the split are described in a later section in this guide.
+Instructions for adjusting the Historical/Middle Manager configs for the split are described in a later section in this guide.
 
 #### Query server
 
@@ -324,7 +324,7 @@ You can copy your existing `coordinator-overlord` configs from the single-server
 
 #### Data
 
-Suppose we are migrating from a single-server deployment that had 32 CPU and 256GiB RAM. In the old deployment, the following configurations for Historicals and MiddleManagers were applied:
+Suppose we are migrating from a single-server deployment that had 32 CPU and 256GiB RAM. In the old deployment, the following configurations for Historicals and Middle Managers were applied:
 
 Historical (Single-server)
 
@@ -334,7 +334,7 @@ druid.processing.numMergeBuffers=8
 druid.processing.numThreads=31
 ```
 
-MiddleManager (Single-server)
+Middle Manager (Single-server)
 
 ```
 druid.worker.capacity=8
@@ -351,7 +351,7 @@ Historical
 - `druid.processing.numMergeBuffers`: Divide the old value from the single-server deployment by the split factor
 - `druid.processing.buffer.sizeBytes`: Keep this unchanged
 
-MiddleManager:
+Middle Manager:
 
 - `druid.worker.capacity`: Divide the old value from the single-server deployment by the split factor
 - `druid.indexer.fork.property.druid.processing.numMergeBuffers`: Keep this unchanged
@@ -368,7 +368,7 @@ druid.processing.numMergeBuffers=4
 druid.processing.numThreads=15
 ```
 
-New MiddleManager (on 2 Data servers)
+New Middle Manager (on 2 Data servers)
 
 ```
 druid.worker.capacity=4
@@ -460,8 +460,8 @@ bin/start-cluster-data-server
 You can add more Data servers as needed.
 
 :::info
- For clusters with complex resource allocation needs, you can break apart Historicals and MiddleManagers and scale the components individually.
- This also allows you take advantage of Druid's built-in MiddleManager autoscaling facility.
+ For clusters with complex resource allocation needs, you can break apart Historicals and Middle Managers and scale the components individually.
+ This also allows you take advantage of Druid's built-in Middle Manager autoscaling facility.
 :::
 
 ## Start Query Server

From ba6f804f4844a6cecaee0fef27577af7828f928e Mon Sep 17 00:00:00 2001
From: Kashif Faraz <kashif.faraz@gmail.com>
Date: Thu, 5 Sep 2024 10:52:01 -0700
Subject: [PATCH 94/99] Fix compaction status API response (#17006)

Description:
#16768 introduces new compaction APIs on the Overlord `/compact/status` and `/compact/progress`.
But the corresponding `OverlordClient` methods do not return an object compatible with the actual
endpoints defined in `OverlordCompactionResource`.

This patch ensures that the objects are compatible.

Changes:
- Add `CompactionStatusResponse` and `CompactionProgressResponse`
- Use these as the return type in `OverlordClient` methods and as the response entity in `OverlordCompactionResource`
- Add `SupervisorCleanupModule` bound on the Coordinator to perform cleanup of supervisors.
Without this module, Coordinator cannot deserialize compaction supervisors.
---
 .../indexing/compact/LocalOverlordClient.java |   7 +-
 .../http/OverlordCompactionResource.java      |  41 ++--
 .../http/OverlordCompactionResourceTest.java  | 232 ++++++++++++++++++
 .../druid/error/DruidExceptionMatcher.java    |   9 +
 .../druid/rpc/indexing/OverlordClient.java    |   7 +-
 .../rpc/indexing/OverlordClientImpl.java      |  11 +-
 .../CompactionProgressResponse.java           |  67 +++++
 .../compaction/CompactionRunSimulator.java    |  11 +-
 .../compaction/CompactionStatusResponse.java  |  69 ++++++
 .../http/CoordinatorCompactionResource.java   |  24 +-
 .../client/indexing/NoopOverlordClient.java   |   7 +-
 .../rpc/indexing/OverlordClientImplTest.java  |  22 +-
 .../CoordinatorCompactionResourceTest.java    |  41 ++--
 .../org/apache/druid/cli/CliCoordinator.java  |  19 +-
 .../druid/guice/SupervisorCleanupModule.java  |  54 ++++
 15 files changed, 528 insertions(+), 93 deletions(-)
 create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionProgressResponse.java
 create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionStatusResponse.java
 create mode 100644 services/src/main/java/org/apache/druid/guice/SupervisorCleanupModule.java

diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/LocalOverlordClient.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/LocalOverlordClient.java
index eeaf138c106..dcce2f4615a 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/LocalOverlordClient.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/LocalOverlordClient.java
@@ -45,7 +45,8 @@ import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.rpc.ServiceRetryPolicy;
 import org.apache.druid.rpc.indexing.OverlordClient;
-import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.compaction.CompactionProgressResponse;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -238,13 +239,13 @@ class LocalOverlordClient implements OverlordClient
   }
 
   @Override
-  public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+  public ListenableFuture<CompactionStatusResponse> getCompactionSnapshots(@Nullable String dataSource)
   {
     throw new UnsupportedOperationException();
   }
 
   @Override
-  public ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource)
+  public ListenableFuture<CompactionProgressResponse> getBytesAwaitingCompaction(String dataSource)
   {
     throw new UnsupportedOperationException();
   }
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResource.java
index 9f21f2ec4c1..425dfc9f35f 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResource.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResource.java
@@ -19,13 +19,18 @@
 
 package org.apache.druid.indexing.overlord.http;
 
-import com.google.common.collect.ImmutableMap;
 import com.google.inject.Inject;
 import com.sun.jersey.spi.container.ResourceFilters;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.error.InvalidInput;
+import org.apache.druid.error.NotFound;
 import org.apache.druid.indexing.compact.CompactionScheduler;
+import org.apache.druid.server.compaction.CompactionProgressResponse;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.apache.druid.server.coordinator.ClusterCompactionConfig;
 import org.apache.druid.server.coordinator.CompactionSupervisorConfig;
+import org.apache.druid.server.http.ServletResourceUtils;
 import org.apache.druid.server.http.security.StateResourceFilter;
 
 import javax.ws.rs.Consumes;
@@ -40,8 +45,8 @@ import java.util.Collection;
 import java.util.Collections;
 
 /**
- * Contains the same logic as {@code CompactionResource} but the APIs are served
- * by {@link CompactionScheduler} instead of {@code DruidCoordinator}.
+ * Contains the same logic as {@code CoordinatorCompactionResource} but the APIs
+ * are served by {@link CompactionScheduler} instead of {@code DruidCoordinator}.
  */
 @Path("/druid/indexer/v1/compaction")
 public class OverlordCompactionResource
@@ -81,18 +86,14 @@ public class OverlordCompactionResource
     }
 
     if (dataSource == null || dataSource.isEmpty()) {
-      return Response.status(Response.Status.BAD_REQUEST)
-                     .entity(Collections.singletonMap("error", "No DataSource specified"))
-                     .build();
+      return ServletResourceUtils.buildErrorResponseFrom(InvalidInput.exception("No DataSource specified"));
     }
 
     final AutoCompactionSnapshot snapshot = scheduler.getCompactionSnapshot(dataSource);
     if (snapshot == null) {
-      return Response.status(Response.Status.NOT_FOUND)
-                     .entity(Collections.singletonMap("error", "Unknown DataSource"))
-                     .build();
+      return ServletResourceUtils.buildErrorResponseFrom(NotFound.exception("Unknown DataSource"));
     } else {
-      return Response.ok(Collections.singletonMap("remainingSegmentSize", snapshot.getBytesAwaitingCompaction()))
+      return Response.ok(new CompactionProgressResponse(snapshot.getBytesAwaitingCompaction()))
                      .build();
     }
   }
@@ -115,13 +116,11 @@ public class OverlordCompactionResource
     } else {
       AutoCompactionSnapshot autoCompactionSnapshot = scheduler.getCompactionSnapshot(dataSource);
       if (autoCompactionSnapshot == null) {
-        return Response.status(Response.Status.NOT_FOUND)
-                       .entity(Collections.singletonMap("error", "Unknown DataSource"))
-                       .build();
+        return ServletResourceUtils.buildErrorResponseFrom(NotFound.exception("Unknown DataSource"));
       }
       snapshots = Collections.singleton(autoCompactionSnapshot);
     }
-    return Response.ok(Collections.singletonMap("latestStatus", snapshots)).build();
+    return Response.ok(new CompactionStatusResponse(snapshots)).build();
   }
 
   @POST
@@ -139,12 +138,12 @@ public class OverlordCompactionResource
 
   private Response buildErrorResponseIfSchedulerDisabled()
   {
-    return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(
-        ImmutableMap.of(
-            "error",
-            "Compaction Supervisors are disabled on the Overlord."
-            + " Use Coordinator APIs to fetch compaction status."
-        )
-    ).build();
+    final String msg = "Compaction Supervisors are disabled on the Overlord."
+                       + " Use Coordinator APIs to fetch compaction status.";
+    return ServletResourceUtils.buildErrorResponseFrom(
+        DruidException.forPersona(DruidException.Persona.USER)
+                      .ofCategory(DruidException.Category.UNSUPPORTED)
+                      .build(msg)
+    );
   }
 }
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java
new file mode 100644
index 00000000000..b93e6e7c1ac
--- /dev/null
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.http;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.error.DruidExceptionMatcher;
+import org.apache.druid.error.ErrorResponse;
+import org.apache.druid.indexing.compact.CompactionScheduler;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.server.compaction.CompactionProgressResponse;
+import org.apache.druid.server.compaction.CompactionStatistics;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.coordinator.CompactionSupervisorConfig;
+import org.easymock.EasyMock;
+import org.hamcrest.MatcherAssert;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.Response;
+import java.util.Collections;
+import java.util.Map;
+
+public class OverlordCompactionResourceTest
+{
+  private static final CompactionSupervisorConfig SUPERVISOR_ENABLED
+      = new CompactionSupervisorConfig(true);
+  private static final CompactionSupervisorConfig SUPERVISOR_DISABLED
+      = new CompactionSupervisorConfig(false);
+
+  private CompactionScheduler scheduler;
+
+  @Before
+  public void setUp()
+  {
+    scheduler = EasyMock.createStrictMock(CompactionScheduler.class);
+  }
+
+  @After
+  public void tearDown()
+  {
+    EasyMock.verify(scheduler);
+  }
+
+  @Test
+  public void testGetCompactionSnapshotWithEmptyDatasource()
+  {
+    final Map<String, AutoCompactionSnapshot> allSnapshots = ImmutableMap.of(
+        TestDataSource.WIKI,
+        AutoCompactionSnapshot.builder(TestDataSource.WIKI).build()
+    );
+
+    EasyMock.expect(scheduler.getAllCompactionSnapshots())
+            .andReturn(allSnapshots).once();
+    EasyMock.replay(scheduler);
+
+    final Response response = new OverlordCompactionResource(SUPERVISOR_ENABLED, scheduler)
+        .getCompactionSnapshots("");
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(
+        new CompactionStatusResponse(allSnapshots.values()),
+        response.getEntity()
+    );
+  }
+
+  @Test
+  public void testGetCompactionSnapshotWithNullDatasource()
+  {
+    final Map<String, AutoCompactionSnapshot> allSnapshots = ImmutableMap.of(
+        TestDataSource.WIKI,
+        AutoCompactionSnapshot.builder(TestDataSource.WIKI).build()
+    );
+
+    EasyMock.expect(scheduler.getAllCompactionSnapshots())
+            .andReturn(allSnapshots).once();
+    EasyMock.replay(scheduler);
+
+    final Response response = new OverlordCompactionResource(SUPERVISOR_ENABLED, scheduler)
+        .getCompactionSnapshots(null);
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(
+        new CompactionStatusResponse(allSnapshots.values()),
+        response.getEntity()
+    );
+  }
+
+  @Test
+  public void testGetCompactionSnapshotWithValidDatasource()
+  {
+    final AutoCompactionSnapshot snapshot = AutoCompactionSnapshot.builder(TestDataSource.WIKI).build();
+
+    EasyMock.expect(scheduler.getCompactionSnapshot(TestDataSource.WIKI))
+            .andReturn(snapshot).once();
+    EasyMock.replay(scheduler);
+
+    final Response response = new OverlordCompactionResource(SUPERVISOR_ENABLED, scheduler)
+        .getCompactionSnapshots(TestDataSource.WIKI);
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(
+        new CompactionStatusResponse(Collections.singleton(snapshot)),
+        response.getEntity()
+    );
+  }
+
+  @Test
+  public void testGetCompactionSnapshotWithInvalidDatasource()
+  {
+    EasyMock.expect(scheduler.getCompactionSnapshot(TestDataSource.KOALA))
+            .andReturn(null).once();
+    EasyMock.replay(scheduler);
+
+    final Response response = new OverlordCompactionResource(SUPERVISOR_ENABLED, scheduler)
+        .getCompactionSnapshots(TestDataSource.KOALA);
+    Assert.assertEquals(404, response.getStatus());
+  }
+
+  @Test
+  public void testGetProgressForValidDatasource()
+  {
+    final AutoCompactionSnapshot.Builder snapshotBuilder
+        = AutoCompactionSnapshot.builder(TestDataSource.WIKI);
+    snapshotBuilder.incrementWaitingStats(CompactionStatistics.create(100L, 10L, 1L));
+    final AutoCompactionSnapshot snapshot = snapshotBuilder.build();
+
+    EasyMock.expect(scheduler.getCompactionSnapshot(TestDataSource.WIKI))
+            .andReturn(snapshot).once();
+    EasyMock.replay(scheduler);
+
+    final Response response = new OverlordCompactionResource(SUPERVISOR_ENABLED, scheduler)
+        .getCompactionProgress(TestDataSource.WIKI);
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(new CompactionProgressResponse(100L), response.getEntity());
+  }
+
+  @Test
+  public void testGetProgressForNullDatasourceReturnsBadRequest()
+  {
+    EasyMock.replay(scheduler);
+
+    final Response response = new OverlordCompactionResource(SUPERVISOR_ENABLED, scheduler)
+        .getCompactionProgress(null);
+    Assert.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus());
+
+    final Object responseEntity = response.getEntity();
+    Assert.assertTrue(responseEntity instanceof ErrorResponse);
+
+    MatcherAssert.assertThat(
+        ((ErrorResponse) responseEntity).getUnderlyingException(),
+        DruidExceptionMatcher.invalidInput().expectMessageIs("No DataSource specified")
+    );
+  }
+
+  @Test
+  public void testGetProgressForInvalidDatasourceReturnsNotFound()
+  {
+    EasyMock.expect(scheduler.getCompactionSnapshot(TestDataSource.KOALA))
+            .andReturn(null).once();
+    EasyMock.replay(scheduler);
+
+    final Response response = new OverlordCompactionResource(SUPERVISOR_ENABLED, scheduler)
+        .getCompactionProgress(TestDataSource.KOALA);
+    Assert.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus());
+
+    final Object responseEntity = response.getEntity();
+    Assert.assertTrue(responseEntity instanceof ErrorResponse);
+
+    MatcherAssert.assertThat(
+        ((ErrorResponse) responseEntity).getUnderlyingException(),
+        DruidExceptionMatcher.notFound().expectMessageIs("Unknown DataSource")
+    );
+  }
+
+  @Test
+  public void testGetProgressReturnsUnsupportedWhenSupervisorDisabled()
+  {
+    EasyMock.replay(scheduler);
+    verifyResponseWhenSupervisorDisabled(
+        new OverlordCompactionResource(SUPERVISOR_DISABLED, scheduler)
+            .getCompactionProgress(TestDataSource.WIKI)
+    );
+  }
+
+  @Test
+  public void testGetSnapshotReturnsUnsupportedWhenSupervisorDisabled()
+  {
+    EasyMock.replay(scheduler);
+    verifyResponseWhenSupervisorDisabled(
+        new OverlordCompactionResource(SUPERVISOR_DISABLED, scheduler)
+            .getCompactionSnapshots(TestDataSource.WIKI)
+    );
+  }
+
+  private void verifyResponseWhenSupervisorDisabled(Response response)
+  {
+    Assert.assertEquals(501, response.getStatus());
+
+    final Object responseEntity = response.getEntity();
+    Assert.assertTrue(responseEntity instanceof ErrorResponse);
+
+    MatcherAssert.assertThat(
+        ((ErrorResponse) responseEntity).getUnderlyingException(),
+        new DruidExceptionMatcher(
+            DruidException.Persona.USER,
+            DruidException.Category.UNSUPPORTED,
+            "general"
+        ).expectMessageIs(
+            "Compaction Supervisors are disabled on the Overlord."
+            + " Use Coordinator APIs to fetch compaction status."
+        )
+    );
+  }
+}
diff --git a/processing/src/test/java/org/apache/druid/error/DruidExceptionMatcher.java b/processing/src/test/java/org/apache/druid/error/DruidExceptionMatcher.java
index 442cbaa48d9..64a24d53626 100644
--- a/processing/src/test/java/org/apache/druid/error/DruidExceptionMatcher.java
+++ b/processing/src/test/java/org/apache/druid/error/DruidExceptionMatcher.java
@@ -40,6 +40,15 @@ public class DruidExceptionMatcher extends DiagnosingMatcher<Throwable>
     );
   }
 
+  public static DruidExceptionMatcher notFound()
+  {
+    return new DruidExceptionMatcher(
+        DruidException.Persona.USER,
+        DruidException.Category.NOT_FOUND,
+        "notFound"
+    );
+  }
+
   public static DruidExceptionMatcher invalidSqlInput()
   {
     return invalidInput().expectContext("sourceType", "sql");
diff --git a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClient.java b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClient.java
index 6797a00b432..fcc93d39c8d 100644
--- a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClient.java
+++ b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClient.java
@@ -35,7 +35,8 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorStatus;
 import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.rpc.ServiceRetryPolicy;
-import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.compaction.CompactionProgressResponse;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -226,7 +227,7 @@ public interface OverlordClient
    * <p>
    * API: {@code /druid/indexer/v1/compaction/progress}
    */
-  ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource);
+  ListenableFuture<CompactionProgressResponse> getBytesAwaitingCompaction(String dataSource);
 
   /**
    * Gets the latest compaction snapshots of one or all datasources.
@@ -236,7 +237,7 @@ public interface OverlordClient
    * @param dataSource If passed as non-null, then the returned list contains only
    *                   the snapshot for this datasource.
    */
-  ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource);
+  ListenableFuture<CompactionStatusResponse> getCompactionSnapshots(@Nullable String dataSource);
 
   /**
    * Returns a copy of this client with a different retry policy.
diff --git a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java
index 2a48e0ed692..1653962e194 100644
--- a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java
+++ b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java
@@ -45,7 +45,8 @@ import org.apache.druid.rpc.IgnoreHttpResponseHandler;
 import org.apache.druid.rpc.RequestBuilder;
 import org.apache.druid.rpc.ServiceClient;
 import org.apache.druid.rpc.ServiceRetryPolicy;
-import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.compaction.CompactionProgressResponse;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
 import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.joda.time.Interval;
 
@@ -314,7 +315,7 @@ public class OverlordClientImpl implements OverlordClient
   }
 
   @Override
-  public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+  public ListenableFuture<CompactionStatusResponse> getCompactionSnapshots(@Nullable String dataSource)
   {
     final StringBuilder pathBuilder = new StringBuilder("/druid/indexer/v1/compaction/status");
     if (dataSource != null && !dataSource.isEmpty()) {
@@ -329,13 +330,13 @@ public class OverlordClientImpl implements OverlordClient
         holder -> JacksonUtils.readValue(
             jsonMapper,
             holder.getContent(),
-            new TypeReference<List<AutoCompactionSnapshot>>() {}
+            CompactionStatusResponse.class
         )
     );
   }
 
   @Override
-  public ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource)
+  public ListenableFuture<CompactionProgressResponse> getBytesAwaitingCompaction(String dataSource)
   {
     final String path = "/druid/indexer/v1/compaction/progress?dataSource=" + dataSource;
     return FutureUtils.transform(
@@ -343,7 +344,7 @@ public class OverlordClientImpl implements OverlordClient
             new RequestBuilder(HttpMethod.GET, path),
             new BytesFullResponseHandler()
         ),
-        holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), Long.class)
+        holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), CompactionProgressResponse.class)
     );
   }
 
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionProgressResponse.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionProgressResponse.java
new file mode 100644
index 00000000000..b91e3d27e7c
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionProgressResponse.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Objects;
+
+/**
+ * Response of {@code /compaction/progress} API exposed by Coordinator and
+ * Overlord (when compaction supervisors are enabled).
+ */
+public class CompactionProgressResponse
+{
+  private final long remainingSegmentSize;
+
+  @JsonCreator
+  public CompactionProgressResponse(
+      @JsonProperty("remainingSegmentSize") long remainingSegmentSize
+  )
+  {
+    this.remainingSegmentSize = remainingSegmentSize;
+  }
+
+  @JsonProperty
+  public long getRemainingSegmentSize()
+  {
+    return remainingSegmentSize;
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    CompactionProgressResponse that = (CompactionProgressResponse) o;
+    return remainingSegmentSize == that.remainingSegmentSize;
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hashCode(remainingSegmentSize);
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
index 6e8ec6b4b11..b51777c9e3b 100644
--- a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
@@ -35,7 +35,6 @@ import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.rpc.ServiceRetryPolicy;
 import org.apache.druid.rpc.indexing.OverlordClient;
-import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.apache.druid.server.coordinator.ClusterCompactionConfig;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
 import org.apache.druid.server.coordinator.DruidCompactionConfig;
@@ -109,7 +108,9 @@ public class CompactionRunSimulator
       )
       {
         final CompactionStatus status = candidateSegments.getCurrentStatus();
-        if (status.getState() == CompactionStatus.State.COMPLETE) {
+        if (status == null) {
+          // do nothing
+        } else if (status.getState() == CompactionStatus.State.COMPLETE) {
           compactedIntervals.addRow(
               createRow(candidateSegments, null, null)
           );
@@ -130,7 +131,7 @@ public class CompactionRunSimulator
         // Add a row for each task in order of submission
         final CompactionStatus status = candidateSegments.getCurrentStatus();
         queuedIntervals.addRow(
-            createRow(candidateSegments, taskPayload.getTuningConfig(), status.getReason())
+            createRow(candidateSegments, taskPayload.getTuningConfig(), status == null ? "" : status.getReason())
         );
       }
     };
@@ -285,13 +286,13 @@ public class CompactionRunSimulator
     }
 
     @Override
-    public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+    public ListenableFuture<CompactionStatusResponse> getCompactionSnapshots(@Nullable String dataSource)
     {
       throw new UnsupportedOperationException();
     }
 
     @Override
-    public ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource)
+    public ListenableFuture<CompactionProgressResponse> getBytesAwaitingCompaction(String dataSource)
     {
       throw new UnsupportedOperationException();
     }
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusResponse.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusResponse.java
new file mode 100644
index 00000000000..756da7aa9c4
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusResponse.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.compaction;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+
+import java.util.Collection;
+import java.util.Objects;
+
+/**
+ * Response of {@code /compaction/status} API exposed by Coordinator and
+ * Overlord (when compaction supervisors are enabled).
+ */
+public class CompactionStatusResponse
+{
+  private final Collection<AutoCompactionSnapshot> latestStatus;
+  
+  @JsonCreator
+  public CompactionStatusResponse(
+      @JsonProperty("latestStatus") Collection<AutoCompactionSnapshot> latestStatus
+  )
+  {
+    this.latestStatus = latestStatus;
+  }
+
+  @JsonProperty
+  public Collection<AutoCompactionSnapshot> getLatestStatus()
+  {
+    return latestStatus;
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    CompactionStatusResponse that = (CompactionStatusResponse) o;
+    return Objects.equals(latestStatus, that.latestStatus);
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hashCode(latestStatus);
+  }
+}
diff --git a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionResource.java b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionResource.java
index 015a27435cf..cb146b7ed95 100644
--- a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionResource.java
+++ b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionResource.java
@@ -21,13 +21,17 @@ package org.apache.druid.server.http;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.inject.Inject;
 import com.sun.jersey.spi.container.ResourceFilters;
 import org.apache.druid.common.guava.FutureUtils;
+import org.apache.druid.error.InternalServerError;
+import org.apache.druid.error.InvalidInput;
+import org.apache.druid.error.NotFound;
 import org.apache.druid.rpc.HttpResponseException;
 import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.server.compaction.CompactionProgressResponse;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.apache.druid.server.coordinator.ClusterCompactionConfig;
 import org.apache.druid.server.coordinator.DruidCoordinator;
@@ -82,9 +86,7 @@ public class CoordinatorCompactionResource
   )
   {
     if (dataSource == null || dataSource.isEmpty()) {
-      return Response.status(Response.Status.BAD_REQUEST)
-                     .entity(ImmutableMap.of("error", "No DataSource specified"))
-                     .build();
+      return ServletResourceUtils.buildErrorResponseFrom(InvalidInput.exception("No DataSource specified"));
     }
 
     if (isCompactionSupervisorEnabled()) {
@@ -93,9 +95,9 @@ public class CoordinatorCompactionResource
 
     final AutoCompactionSnapshot snapshot = coordinator.getAutoCompactionSnapshotForDataSource(dataSource);
     if (snapshot == null) {
-      return Response.status(Response.Status.NOT_FOUND).entity(ImmutableMap.of("error", "Unknown DataSource")).build();
+      return ServletResourceUtils.buildErrorResponseFrom(NotFound.exception("Unknown DataSource"));
     } else {
-      return Response.ok(ImmutableMap.of("remainingSegmentSize", snapshot.getBytesAwaitingCompaction())).build();
+      return Response.ok(new CompactionProgressResponse(snapshot.getBytesAwaitingCompaction())).build();
     }
   }
 
@@ -117,11 +119,11 @@ public class CoordinatorCompactionResource
     } else {
       AutoCompactionSnapshot autoCompactionSnapshot = coordinator.getAutoCompactionSnapshotForDataSource(dataSource);
       if (autoCompactionSnapshot == null) {
-        return Response.status(Response.Status.NOT_FOUND).entity(ImmutableMap.of("error", "Unknown DataSource")).build();
+        return ServletResourceUtils.buildErrorResponseFrom(NotFound.exception("Unknown DataSource"));
       }
       snapshots = ImmutableList.of(autoCompactionSnapshot);
     }
-    return Response.ok(ImmutableMap.of("latestStatus", snapshots)).build();
+    return Response.ok(new CompactionStatusResponse(snapshots)).build();
   }
 
   @POST
@@ -149,9 +151,9 @@ public class CoordinatorCompactionResource
                        .entity(cause.getResponse().getContent())
                        .build();
       } else {
-        return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
-                       .entity(ImmutableMap.of("error", e.getMessage()))
-                       .build();
+        return ServletResourceUtils.buildErrorResponseFrom(
+            InternalServerError.exception(e.getMessage())
+        );
       }
     }
   }
diff --git a/server/src/test/java/org/apache/druid/client/indexing/NoopOverlordClient.java b/server/src/test/java/org/apache/druid/client/indexing/NoopOverlordClient.java
index c73ff1ca059..8ccc32e435c 100644
--- a/server/src/test/java/org/apache/druid/client/indexing/NoopOverlordClient.java
+++ b/server/src/test/java/org/apache/druid/client/indexing/NoopOverlordClient.java
@@ -28,7 +28,8 @@ import org.apache.druid.java.util.common.parsers.CloseableIterator;
 import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.rpc.ServiceRetryPolicy;
 import org.apache.druid.rpc.indexing.OverlordClient;
-import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
+import org.apache.druid.server.compaction.CompactionProgressResponse;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
@@ -124,13 +125,13 @@ public class NoopOverlordClient implements OverlordClient
   }
 
   @Override
-  public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+  public ListenableFuture<CompactionStatusResponse> getCompactionSnapshots(@Nullable String dataSource)
   {
     throw new UnsupportedOperationException();
   }
 
   @Override
-  public ListenableFuture<Long> getBytesAwaitingCompaction(String dataSource)
+  public ListenableFuture<CompactionProgressResponse> getBytesAwaitingCompaction(String dataSource)
   {
     throw new UnsupportedOperationException();
   }
diff --git a/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java b/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java
index 18e7e276950..1167d10502d 100644
--- a/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java
+++ b/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java
@@ -47,6 +47,8 @@ import org.apache.druid.metadata.LockFilterPolicy;
 import org.apache.druid.rpc.HttpResponseException;
 import org.apache.druid.rpc.MockServiceClient;
 import org.apache.druid.rpc.RequestBuilder;
+import org.apache.druid.server.compaction.CompactionProgressResponse;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.MatcherAssert;
@@ -477,18 +479,18 @@ public class OverlordClientImplTest
                               .withStatus(AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING)
                               .build(),
         AutoCompactionSnapshot.builder("ds2")
-                              .withStatus(AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING)
+                              .withStatus(AutoCompactionSnapshot.AutoCompactionScheduleStatus.NOT_ENABLED)
                               .build()
     );
     serviceClient.expectAndRespond(
         new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/status"),
         HttpResponseStatus.OK,
         Collections.emptyMap(),
-        DefaultObjectMapper.INSTANCE.writeValueAsBytes(compactionSnapshots)
+        DefaultObjectMapper.INSTANCE.writeValueAsBytes(new CompactionStatusResponse(compactionSnapshots))
     );
 
     Assert.assertEquals(
-        compactionSnapshots,
+        new CompactionStatusResponse(compactionSnapshots),
         overlordClient.getCompactionSnapshots(null).get()
     );
   }
@@ -498,19 +500,17 @@ public class OverlordClientImplTest
       throws JsonProcessingException, ExecutionException, InterruptedException
   {
     final List<AutoCompactionSnapshot> compactionSnapshots = Collections.singletonList(
-        AutoCompactionSnapshot.builder("ds1")
-                              .withStatus(AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING)
-                              .build()
+        AutoCompactionSnapshot.builder("ds1").build()
     );
     serviceClient.expectAndRespond(
         new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/status?dataSource=ds1"),
         HttpResponseStatus.OK,
         Collections.emptyMap(),
-        DefaultObjectMapper.INSTANCE.writeValueAsBytes(compactionSnapshots)
+        DefaultObjectMapper.INSTANCE.writeValueAsBytes(new CompactionStatusResponse(compactionSnapshots))
     );
 
     Assert.assertEquals(
-        compactionSnapshots,
+        new CompactionStatusResponse(compactionSnapshots),
         overlordClient.getCompactionSnapshots("ds1").get()
     );
   }
@@ -523,12 +523,12 @@ public class OverlordClientImplTest
         new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/progress?dataSource=ds1"),
         HttpResponseStatus.OK,
         Collections.emptyMap(),
-        DefaultObjectMapper.INSTANCE.writeValueAsBytes(100_000L)
+        DefaultObjectMapper.INSTANCE.writeValueAsBytes(new CompactionProgressResponse(100_000L))
     );
 
     Assert.assertEquals(
-        100_000L,
-        overlordClient.getBytesAwaitingCompaction("ds1").get().longValue()
+        new CompactionProgressResponse(100_000L),
+        overlordClient.getBytesAwaitingCompaction("ds1").get()
     );
   }
 }
diff --git a/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionResourceTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionResourceTest.java
index 8c6c747d3a8..4a73047d195 100644
--- a/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionResourceTest.java
+++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionResourceTest.java
@@ -19,16 +19,19 @@
 
 package org.apache.druid.server.http;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import org.apache.druid.client.indexing.NoopOverlordClient;
+import org.apache.druid.error.DruidExceptionMatcher;
+import org.apache.druid.error.ErrorResponse;
 import org.apache.druid.rpc.indexing.OverlordClient;
 import org.apache.druid.server.compaction.CompactionStatistics;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
 import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
 import org.apache.druid.server.coordinator.DruidCoordinator;
 import org.easymock.EasyMock;
+import org.hamcrest.MatcherAssert;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -37,7 +40,6 @@ import org.junit.Test;
 import javax.annotation.Nullable;
 import javax.ws.rs.core.Response;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 public class CoordinatorCompactionResourceTest
@@ -92,7 +94,7 @@ public class CoordinatorCompactionResourceTest
 
     final Response response = new CoordinatorCompactionResource(mock, overlordClient)
         .getCompactionSnapshotForDataSource("");
-    Assert.assertEquals(ImmutableMap.of("latestStatus", expected.values()), response.getEntity());
+    Assert.assertEquals(new CompactionStatusResponse(expected.values()), response.getEntity());
     Assert.assertEquals(200, response.getStatus());
   }
 
@@ -110,7 +112,7 @@ public class CoordinatorCompactionResourceTest
 
     final Response response = new CoordinatorCompactionResource(mock, overlordClient)
         .getCompactionSnapshotForDataSource(null);
-    Assert.assertEquals(ImmutableMap.of("latestStatus", expected.values()), response.getEntity());
+    Assert.assertEquals(new CompactionStatusResponse(expected.values()), response.getEntity());
     Assert.assertEquals(200, response.getStatus());
   }
 
@@ -119,12 +121,16 @@ public class CoordinatorCompactionResourceTest
   {
     String dataSourceName = "datasource_1";
 
-    EasyMock.expect(mock.getAutoCompactionSnapshotForDataSource(dataSourceName)).andReturn(expectedSnapshot).once();
+    EasyMock.expect(mock.getAutoCompactionSnapshotForDataSource(dataSourceName))
+            .andReturn(expectedSnapshot).once();
     EasyMock.replay(mock);
 
     final Response response = new CoordinatorCompactionResource(mock, overlordClient)
         .getCompactionSnapshotForDataSource(dataSourceName);
-    Assert.assertEquals(ImmutableMap.of("latestStatus", ImmutableList.of(expectedSnapshot)), response.getEntity());
+    Assert.assertEquals(
+        new CompactionStatusResponse(Collections.singletonList(expectedSnapshot)),
+        response.getEntity()
+    );
     Assert.assertEquals(200, response.getStatus());
   }
 
@@ -133,7 +139,8 @@ public class CoordinatorCompactionResourceTest
   {
     String dataSourceName = "invalid_datasource";
 
-    EasyMock.expect(mock.getAutoCompactionSnapshotForDataSource(dataSourceName)).andReturn(null).once();
+    EasyMock.expect(mock.getAutoCompactionSnapshotForDataSource(dataSourceName))
+            .andReturn(null).once();
     EasyMock.replay(mock);
 
     final Response response = new CoordinatorCompactionResource(mock, overlordClient)
@@ -149,14 +156,18 @@ public class CoordinatorCompactionResourceTest
     final Response response = new CoordinatorCompactionResource(mock, overlordClient)
         .getCompactionProgress(null);
     Assert.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus());
-    Assert.assertEquals(
-        ImmutableMap.of("error", "No DataSource specified"),
-        response.getEntity()
+
+    final Object responseEntity = response.getEntity();
+    Assert.assertTrue(responseEntity instanceof ErrorResponse);
+
+    MatcherAssert.assertThat(
+        ((ErrorResponse) responseEntity).getUnderlyingException(),
+        DruidExceptionMatcher.invalidInput().expectMessageIs("No DataSource specified")
     );
   }
 
   @Test
-  public void testGetSnapshotWhenCompactionSupervisorIsEnabled()
+  public void testGetSnapshotRedirectsToOverlordWhenSupervisorIsEnabled()
   {
     EasyMock.replay(mock);
 
@@ -172,9 +183,11 @@ public class CoordinatorCompactionResourceTest
       }
 
       @Override
-      public ListenableFuture<List<AutoCompactionSnapshot>> getCompactionSnapshots(@Nullable String dataSource)
+      public ListenableFuture<CompactionStatusResponse> getCompactionSnapshots(@Nullable String dataSource)
       {
-        return Futures.immediateFuture(Collections.singletonList(snapshotFromOverlord));
+        return Futures.immediateFuture(
+            new CompactionStatusResponse(Collections.singletonList(snapshotFromOverlord))
+        );
       }
     };
 
@@ -182,7 +195,7 @@ public class CoordinatorCompactionResourceTest
         .getCompactionSnapshotForDataSource(dataSourceName);
     Assert.assertEquals(200, response.getStatus());
     Assert.assertEquals(
-        Collections.singletonList(snapshotFromOverlord),
+        new CompactionStatusResponse(Collections.singletonList(snapshotFromOverlord)),
         response.getEntity()
     );
   }
diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
index 68923f21f1c..ba7bd43ef8d 100644
--- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
+++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java
@@ -51,11 +51,10 @@ import org.apache.druid.guice.LazySingleton;
 import org.apache.druid.guice.LifecycleModule;
 import org.apache.druid.guice.ManageLifecycle;
 import org.apache.druid.guice.QueryableModule;
+import org.apache.druid.guice.SupervisorCleanupModule;
 import org.apache.druid.guice.annotations.EscalatedGlobal;
 import org.apache.druid.guice.annotations.Global;
 import org.apache.druid.guice.http.JettyHttpClientModule;
-import org.apache.druid.indexing.overlord.TaskMaster;
-import org.apache.druid.indexing.overlord.TaskStorage;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
@@ -89,7 +88,6 @@ import org.apache.druid.query.metadata.SegmentMetadataQueryConfig;
 import org.apache.druid.query.metadata.SegmentMetadataQueryQueryToolChest;
 import org.apache.druid.query.metadata.SegmentMetadataQueryRunnerFactory;
 import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
-import org.apache.druid.segment.incremental.RowIngestionMetersFactory;
 import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
 import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache;
 import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig;
@@ -276,16 +274,6 @@ public class CliCoordinator extends ServerRunnable
             LifecycleModule.register(binder, Server.class);
             LifecycleModule.register(binder, DataSourcesResource.class);
 
-            if (properties.containsKey("druid.coordinator.merge.on")) {
-              throw new UnsupportedOperationException(
-                  "'druid.coordinator.merge.on' is not supported anymore. "
-                  + "Please consider using Coordinator's automatic compaction instead. "
-                  + "See https://druid.apache.org/docs/latest/operations/segment-optimization.html and "
-                  + "https://druid.apache.org/docs/latest/api-reference/api-reference.html#compaction-configuration "
-                  + "for more details about compaction."
-              );
-            }
-
             bindAnnouncer(
                 binder,
                 Coordinator.class,
@@ -296,10 +284,6 @@ public class CliCoordinator extends ServerRunnable
             LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class));
 
             if (!beOverlord) {
-              // These are needed to deserialize SupervisorSpec for Supervisor Auto Cleanup
-              binder.bind(TaskStorage.class).toProvider(Providers.of(null));
-              binder.bind(TaskMaster.class).toProvider(Providers.of(null));
-              binder.bind(RowIngestionMetersFactory.class).toProvider(Providers.of(null));
               // Bind HeartbeatSupplier only when the service operates independently of Overlord.
               binder.bind(new TypeLiteral<Supplier<Map<String, Object>>>() {})
                   .annotatedWith(Names.named(ServiceStatusMonitor.HEARTBEAT_TAGS_BINDING))
@@ -342,6 +326,7 @@ public class CliCoordinator extends ServerRunnable
       // Only add LookupSerdeModule if !beOverlord, since CliOverlord includes it, and having two copies causes
       // the injector to get confused due to having multiple bindings for the same classes.
       modules.add(new LookupSerdeModule());
+      modules.add(new SupervisorCleanupModule());
     }
 
     return modules;
diff --git a/services/src/main/java/org/apache/druid/guice/SupervisorCleanupModule.java b/services/src/main/java/org/apache/druid/guice/SupervisorCleanupModule.java
new file mode 100644
index 00000000000..c60442a06f7
--- /dev/null
+++ b/services/src/main/java/org/apache/druid/guice/SupervisorCleanupModule.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.guice;
+
+import com.fasterxml.jackson.databind.Module;
+import com.google.inject.Binder;
+import com.google.inject.util.Providers;
+import org.apache.druid.indexing.compact.CompactionScheduler;
+import org.apache.druid.indexing.overlord.TaskMaster;
+import org.apache.druid.indexing.overlord.TaskStorage;
+import org.apache.druid.initialization.DruidModule;
+import org.apache.druid.segment.incremental.RowIngestionMetersFactory;
+
+import java.util.List;
+
+/**
+ * Contains bindings necessary for Coordinator to perform supervisor cleanup
+ * when Coordinator and Overlord are running as separate processes.
+ */
+public class SupervisorCleanupModule implements DruidModule
+{
+  @Override
+  public void configure(Binder binder)
+  {
+    // These are needed to deserialize SupervisorSpec for Supervisor Auto Cleanup
+    binder.bind(TaskStorage.class).toProvider(Providers.of(null));
+    binder.bind(TaskMaster.class).toProvider(Providers.of(null));
+    binder.bind(RowIngestionMetersFactory.class).toProvider(Providers.of(null));
+    binder.bind(CompactionScheduler.class).toProvider(Providers.of(null));
+  }
+
+  @Override
+  public List<? extends Module> getJacksonModules()
+  {
+    return new SupervisorModule().getJacksonModules();
+  }
+}

From dc5c55a836a62aab6e2dd91884f186703b953645 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Thu, 5 Sep 2024 13:51:03 -0700
Subject: [PATCH 95/99] Web console: better tooltip when no size is available
 (#17008)

* better tooltip when no size is available

* better labels for columns

* fix label in segments view
---
 .../datasources-view.spec.tsx.snap            | 30 +++++++++---
 .../datasources-view/datasources-view.tsx     | 46 +++++++++----------
 .../__snapshots__/segments-view.spec.tsx.snap |  5 +-
 .../src/views/segments-view/segments-view.tsx | 10 ++--
 .../src/views/services-view/services-view.tsx |  5 +-
 .../execution-stages-pane.tsx                 | 17 +++----
 6 files changed, 66 insertions(+), 47 deletions(-)

diff --git a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
index 47dd15ff05e..2444d1a4f23 100644
--- a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
+++ b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
@@ -67,17 +67,35 @@ exports[`DatasourcesView matches snapshot 1`] = `
           "Availability",
           "Historical load/drop queues",
           "Total data size",
-          "Running tasks",
+          {
+            "label": "sys.tasks",
+            "text": "Running tasks",
+          },
           "Segment rows",
           "Segment size",
-          "Segment granularity",
+          {
+            "label": "𝑓(sys.segments)",
+            "text": "Segment granularity",
+          },
           "Total rows",
           "Avg. row size",
           "Replicated size",
-          "Compaction",
-          "% Compacted",
-          "Left to be compacted",
-          "Retention",
+          {
+            "label": "compaction API",
+            "text": "Compaction",
+          },
+          {
+            "label": "compaction API",
+            "text": "% Compacted",
+          },
+          {
+            "label": "compaction API",
+            "text": "Left to be compacted",
+          },
+          {
+            "label": "rules API",
+            "text": "Retention",
+          },
         ]
       }
       onChange={[Function]}
diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx
index 69c6302c654..96a32c545ab 100644
--- a/web-console/src/views/datasources-view/datasources-view.tsx
+++ b/web-console/src/views/datasources-view/datasources-view.tsx
@@ -26,6 +26,7 @@ import type { Filter } from 'react-table';
 import ReactTable from 'react-table';
 
 import {
+  type TableColumnSelectorColumn,
   ACTION_COLUMN_ID,
   ACTION_COLUMN_LABEL,
   ACTION_COLUMN_WIDTH,
@@ -86,44 +87,44 @@ import { RuleUtil } from '../../utils/load-rule';
 
 import './datasources-view.scss';
 
-const tableColumns: Record<CapabilitiesMode, string[]> = {
+const TABLE_COLUMNS_BY_MODE: Record<CapabilitiesMode, TableColumnSelectorColumn[]> = {
   'full': [
     'Datasource name',
     'Availability',
     'Historical load/drop queues',
     'Total data size',
-    'Running tasks',
+    { text: 'Running tasks', label: 'sys.tasks' },
     'Segment rows',
     'Segment size',
-    'Segment granularity',
+    { text: 'Segment granularity', label: '𝑓(sys.segments)' },
     'Total rows',
     'Avg. row size',
     'Replicated size',
-    'Compaction',
-    '% Compacted',
-    'Left to be compacted',
-    'Retention',
+    { text: 'Compaction', label: 'compaction API' },
+    { text: '% Compacted', label: 'compaction API' },
+    { text: 'Left to be compacted', label: 'compaction API' },
+    { text: 'Retention', label: 'rules API' },
   ],
   'no-sql': [
     'Datasource name',
     'Availability',
     'Historical load/drop queues',
     'Total data size',
-    'Running tasks',
-    'Compaction',
-    '% Compacted',
-    'Left to be compacted',
-    'Retention',
+    { text: 'Running tasks', label: 'tasks API' },
+    { text: 'Compaction', label: 'compaction API' },
+    { text: '% Compacted', label: 'compaction API' },
+    { text: 'Left to be compacted', label: 'compaction API' },
+    { text: 'Retention', label: 'rules API' },
   ],
   'no-proxy': [
     'Datasource name',
     'Availability',
     'Historical load/drop queues',
     'Total data size',
-    'Running tasks',
+    { text: 'Running tasks', label: 'sys.tasks' },
     'Segment rows',
     'Segment size',
-    'Segment granularity',
+    { text: 'Segment granularity', label: '𝑓(sys.segments)' },
     'Total rows',
     'Avg. row size',
     'Replicated size',
@@ -517,21 +518,18 @@ GROUP BY 1, 2`;
           if (capabilities.hasOverlordAccess()) {
             auxiliaryQueries.push(async (datasourcesAndDefaultRules, cancelToken) => {
               try {
-                const runningTasks = await queryDruidSql<RunningTaskRow>(
-                  {
-                    query: DatasourcesView.RUNNING_TASK_SQL,
-                  },
-                  cancelToken,
-                );
+                const taskList = (
+                  await Api.instance.get(`/druid/indexer/v1/tasks?state=running`, { cancelToken })
+                ).data;
 
                 const runningTasksByDatasource = groupByAsMap(
-                  runningTasks,
-                  x => x.datasource,
+                  taskList,
+                  (t: any) => t.dataSource,
                   xs =>
                     groupByAsMap(
                       xs,
                       x => normalizeTaskType(x.type),
-                      ys => sum(ys, y => y.num_running_tasks),
+                      ys => ys.length,
                     ),
                 );
 
@@ -1703,7 +1701,7 @@ GROUP BY 1, 2`;
             disabled={!capabilities.hasSqlOrCoordinatorAccess()}
           />
           <TableColumnSelector
-            columns={tableColumns[capabilities.getMode()]}
+            columns={TABLE_COLUMNS_BY_MODE[capabilities.getMode()]}
             onChange={column =>
               this.setState(prevState => ({
                 visibleColumns: prevState.visibleColumns.toggle(column),
diff --git a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
index 595a48f743e..e91e6d44eaf 100755
--- a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
+++ b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
@@ -55,7 +55,10 @@ exports[`SegmentsView matches snapshot 1`] = `
             "Start",
             "End",
             "Version",
-            "Time span",
+            {
+              "label": "𝑓(sys.segments)",
+              "text": "Time span",
+            },
             "Shard type",
             "Shard spec",
             "Partition",
diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx
index c266e7bd994..def5b2baba9 100644
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@@ -26,6 +26,7 @@ import type { Filter } from 'react-table';
 import ReactTable from 'react-table';
 
 import {
+  type TableColumnSelectorColumn,
   ACTION_COLUMN_ID,
   ACTION_COLUMN_LABEL,
   ACTION_COLUMN_WIDTH,
@@ -76,14 +77,14 @@ import type { BasicAction } from '../../utils/basic-action';
 
 import './segments-view.scss';
 
-const tableColumns: Record<CapabilitiesMode, string[]> = {
+const TABLE_COLUMNS_BY_MODE: Record<CapabilitiesMode, TableColumnSelectorColumn[]> = {
   'full': [
     'Segment ID',
     'Datasource',
     'Start',
     'End',
     'Version',
-    'Time span',
+    { text: 'Time span', label: '𝑓(sys.segments)' },
     'Shard type',
     'Shard spec',
     'Partition',
@@ -105,6 +106,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
     'Start',
     'End',
     'Version',
+    { text: 'Time span', label: '𝑓(sys.segments)' },
     'Shard type',
     'Shard spec',
     'Partition',
@@ -260,7 +262,7 @@ END AS "time_span"`,
         if (capabilities.hasSql()) {
           const whereParts = filterMap(filtered, (f: Filter) => {
             if (f.id === 'shard_type') {
-              // Special handling for shard_type that needs to be search in the shard_spec
+              // Special handling for shard_type that needs to be searched for in the shard_spec
               // Creates filters like `shard_spec LIKE '%"type":"numbered"%'`
               const modeAndNeedle = parseFilterModeAndNeedle(f);
               if (!modeAndNeedle) return;
@@ -1008,7 +1010,7 @@ END AS "time_span"`,
               disabled={!capabilities.hasSqlOrCoordinatorAccess()}
             />
             <TableColumnSelector
-              columns={tableColumns[capabilities.getMode()]}
+              columns={TABLE_COLUMNS_BY_MODE[capabilities.getMode()]}
               onChange={column =>
                 this.setState(prevState => ({
                   visibleColumns: prevState.visibleColumns.toggle(column),
diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx
index 86693f85fcf..ebff85dc81f 100644
--- a/web-console/src/views/services-view/services-view.tsx
+++ b/web-console/src/views/services-view/services-view.tsx
@@ -24,6 +24,7 @@ import type { Filter } from 'react-table';
 import ReactTable from 'react-table';
 
 import {
+  type TableColumnSelectorColumn,
   ACTION_COLUMN_ID,
   ACTION_COLUMN_LABEL,
   ACTION_COLUMN_WIDTH,
@@ -59,7 +60,7 @@ import type { BasicAction } from '../../utils/basic-action';
 
 import './services-view.scss';
 
-const tableColumns: Record<CapabilitiesMode, string[]> = {
+const TABLE_COLUMNS_BY_MODE: Record<CapabilitiesMode, TableColumnSelectorColumn[]> = {
   'full': [
     'Service',
     'Type',
@@ -804,7 +805,7 @@ ORDER BY
           />
           {this.renderBulkServicesActions()}
           <TableColumnSelector
-            columns={tableColumns[capabilities.getMode()]}
+            columns={TABLE_COLUMNS_BY_MODE[capabilities.getMode()]}
             onChange={column =>
               this.setState(prevState => ({
                 visibleColumns: prevState.visibleColumns.toggle(column),
diff --git a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
index e8ee6d88b20..8cbe89ee7bb 100644
--- a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
+++ b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
@@ -61,6 +61,7 @@ import './execution-stages-pane.scss';
 const MAX_STAGE_ROWS = 20;
 const MAX_DETAIL_ROWS = 20;
 const NOT_SIZE_ON_DISK = '(does not represent size on disk)';
+const NO_SIZE_INFO = 'no size info';
 
 function summarizeTableInput(tableStageInput: StageInput): string {
   if (tableStageInput.type !== 'table') return '';
@@ -271,7 +272,7 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
                       title={
                         c.bytes
                           ? `Uncompressed size: ${formatBytesCompact(c.bytes)} ${NOT_SIZE_ON_DISK}`
-                          : undefined
+                          : NO_SIZE_INFO
                       }
                     />
                     {Boolean(c.totalFiles) && (
@@ -378,7 +379,7 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
                     title={
                       c.bytes
                         ? `Uncompressed size: ${formatBytesCompact(c.bytes)} ${NOT_SIZE_ON_DISK}`
-                        : undefined
+                        : NO_SIZE_INFO
                     }
                   />
                 );
@@ -395,19 +396,15 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
     const hasCounter = stages.hasCounterForStage(stage, inputCounter);
     const bytes = stages.getTotalCounterForStage(stage, inputCounter, 'bytes');
     const inputFileCount = stages.getTotalCounterForStage(stage, inputCounter, 'totalFiles');
+    const inputLabel = `${formatInputLabel(stage, inputNumber)} (input${inputNumber})`;
     return (
       <div
         className="data-transfer"
         key={inputNumber}
         title={
           bytes
-            ? `${formatInputLabel(
-                stage,
-                inputNumber,
-              )} (input${inputNumber}) uncompressed size: ${formatBytesCompact(
-                bytes,
-              )} ${NOT_SIZE_ON_DISK}`
-            : undefined
+            ? `${inputLabel} uncompressed size: ${formatBytesCompact(bytes)} ${NOT_SIZE_ON_DISK}`
+            : `${inputLabel}: ${NO_SIZE_INFO}`
         }
       >
         <BracedText
@@ -510,7 +507,7 @@ ${title} uncompressed size: ${formatBytesCompact(
     if (!stages.hasCounterForStage(stage, 'segmentGenerationProgress')) return;
 
     return (
-      <div className="data-transfer">
+      <div className="data-transfer" title={NO_SIZE_INFO}>
         <BracedText
           text={formatRows(stages.getTotalSegmentGenerationProgressForStage(stage, field))}
           braces={rowsValues}

From c49dc83b22e19b3fcdb0c7ac81896a32f0e5015e Mon Sep 17 00:00:00 2001
From: Edgar Melendrez <edgar.melendrez@imply.io>
Date: Thu, 5 Sep 2024 17:02:45 -0700
Subject: [PATCH 96/99] [docs] batch 12: reduction functions (#16930)

* [docs] batch 12: reduction functions

* Update docs/querying/sql-functions.md

* Update docs/querying/sql-functions.md

* Update docs/querying/sql-functions.md

* applying suggestions

* Apply suggestions from code review

Co-authored-by: Katya Macedo  <38017980+ektravel@users.noreply.github.com>

---------

Co-authored-by: Benedict Jin <asdf2014@apache.org>
Co-authored-by: Katya Macedo <38017980+ektravel@users.noreply.github.com>
---
 docs/querying/sql-functions.md | 48 +++++++++++++++++++++++++++++-----
 1 file changed, 42 insertions(+), 6 deletions(-)

diff --git a/docs/querying/sql-functions.md b/docs/querying/sql-functions.md
index abb17434a8e..72ed63c6d7c 100644
--- a/docs/querying/sql-functions.md
+++ b/docs/querying/sql-functions.md
@@ -1211,11 +1211,29 @@ Returns the following:
 
 ## GREATEST
 
-`GREATEST([expr1, ...])`
+Returns the maximum value from the provided expressions. For information on how Druid interprets the arguments passed into the function, see [Reduction functions](sql-scalar.md#reduction-functions).
 
-**Function type:** [Scalar, reduction](sql-scalar.md#reduction-functions)
+* **Syntax:** `GREATEST([expr1, ...])`
+* **Function type:** Scalar, reduction
+
+<details><summary>Example</summary>
+
+The following example returns the greatest value between the numeric constant `PI`, the integer number `4`, and the double `-5.0`. Druid interprets these arguments as DOUBLE data type.
+
+```sql
+SELECT GREATEST(PI, 4, -5.0) AS "greatest"
+```
+
+Returns the following:
+
+| `greatest` |
+| -- | 
+| `4` |
+
+</details>
+
+[Learn more](sql-scalar.md#reduction-functions)
 
-Returns the maximum value from the provided arguments.
 
 ## GROUPING
 
@@ -1483,11 +1501,29 @@ If you do not supply an `offset`, returns the value evaluated at the row followi
 
 ## LEAST
 
-`LEAST([expr1, ...])`
+Returns the minimum value from the provided expressions. For information on how Druid interprets the arguments passed into the function, see [Reduction functions](sql-scalar.md#reduction-functions).
 
-**Function type:** [Scalar, reduction](sql-scalar.md#reduction-functions)
+* **Syntax:** `LEAST([expr1, ...])`
+* **Function type:** Scalar, reduction
+
+<details><summary>Example</summary>
+
+The following example returns the minimum value between the strings `apple`, `orange`, and `pear`. Druid interprets these arguments as STRING data type. 
+
+```sql
+SELECT LEAST( 'apple', 'orange', 'pear') AS "least"
+```
+
+Returns the following:
+
+| `least` |
+| -- |
+| `apple` |
+
+</details>
+
+[Learn more](sql-scalar.md#reduction-functions)
 
-Returns the minimum value from the provided arguments.
 
 ## LEFT
 

From 175636b28ffa296eae36eea57b6a46f358fca25b Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Thu, 5 Sep 2024 17:20:00 -0700
Subject: [PATCH 97/99] Frame writers: Coerce numeric and array types in
 certain cases. (#16994)

This patch adds "TypeCastSelectors", which is used when writing frames to
perform two coercions:

- When a numeric type is desired and the underlying type is non-numeric or
  unknown, the underlying selector is wrapped, "getObject" is called and the
  result is coerced using "ExprEval.ofType". This differs from the prior
  behavior where the primitive methods like "getLong", "getDouble", etc, would
  be called directly. This fixes an issue where a column would be read as
  all-zeroes when its SQL type is numeric and its physical type is string, which
  can happen when evolving a column's type from string to number.

-  When an array type is desired, the underlying selector is wrapped,
   "getObject" is called, and the result is coerced to Object[]. This coercion
   replaces some earlier logic from #15917.
---
 .../GroupByPostShuffleFrameProcessor.java     |   2 +-
 ...ssignmentWithValidationDisabled_select.sql |   2 +-
 .../druid/frame/field/FieldWriters.java       |  25 +-
 .../field/NumericArrayFieldSelector.java      |  10 +-
 .../frame/field/NumericArrayFieldWriter.java  |  22 +-
 .../druid/frame/write/FrameWriterUtils.java   |  55 +--
 .../ObjectToArrayColumnValueSelector.java     | 101 +++++
 .../ObjectToNumberColumnValueSelector.java    | 135 ++++++
 .../frame/write/cast/TypeCastSelectors.java   | 121 ++++++
 .../write/columnar/FrameColumnWriters.java    |  25 +-
 .../NumericArrayFrameColumnWriter.java        |  11 +-
 .../org/apache/druid/math/expr/ExprEval.java  |  39 +-
 .../field/StringArrayFieldWriterTest.java     |   5 +-
 .../frame/field/StringFieldReaderTest.java    |   5 +-
 .../write/cast/TypeCastSelectorsTest.java     | 394 ++++++++++++++++++
 15 files changed, 835 insertions(+), 117 deletions(-)
 create mode 100644 processing/src/main/java/org/apache/druid/frame/write/cast/ObjectToArrayColumnValueSelector.java
 create mode 100644 processing/src/main/java/org/apache/druid/frame/write/cast/ObjectToNumberColumnValueSelector.java
 create mode 100644 processing/src/main/java/org/apache/druid/frame/write/cast/TypeCastSelectors.java
 create mode 100644 processing/src/test/java/org/apache/druid/frame/write/cast/TypeCastSelectorsTest.java

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByPostShuffleFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByPostShuffleFrameProcessor.java
index 7861a45f61d..5b0a3ddefd2 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByPostShuffleFrameProcessor.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/groupby/GroupByPostShuffleFrameProcessor.java
@@ -109,7 +109,7 @@ public class GroupByPostShuffleFrameProcessor implements FrameProcessor<Object>
             RowBasedGrouperHelper.createResultRowBasedColumnSelectorFactory(
                 query,
                 () -> outputRow,
-                RowSignature.Finalization.YES
+                GroupByQueryKit.isFinalize(query) ? RowSignature.Finalization.YES : RowSignature.Finalization.NO
             )
         );
   }
diff --git a/integration-tests-ex/cases/src/test/resources/catalog/incompatibleTypeAssignmentWithValidationDisabled_select.sql b/integration-tests-ex/cases/src/test/resources/catalog/incompatibleTypeAssignmentWithValidationDisabled_select.sql
index 4b6fafdae91..ef3e474736c 100644
--- a/integration-tests-ex/cases/src/test/resources/catalog/incompatibleTypeAssignmentWithValidationDisabled_select.sql
+++ b/integration-tests-ex/cases/src/test/resources/catalog/incompatibleTypeAssignmentWithValidationDisabled_select.sql
@@ -4,7 +4,7 @@
     "expectedResults": [
       {
         "__time": 1672058096000,
-        "double_col": 0.0
+        "double_col": null
       }
     ]
   }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/FieldWriters.java b/processing/src/main/java/org/apache/druid/frame/field/FieldWriters.java
index 028c9fd39c5..dde61a4d2ea 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/FieldWriters.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/FieldWriters.java
@@ -22,6 +22,7 @@ package org.apache.druid.frame.field;
 import org.apache.druid.frame.key.RowKey;
 import org.apache.druid.frame.write.RowBasedFrameWriterFactory;
 import org.apache.druid.frame.write.UnsupportedColumnTypeException;
+import org.apache.druid.frame.write.cast.TypeCastSelectors;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.segment.ColumnSelectorFactory;
@@ -101,7 +102,8 @@ public class FieldWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.LONG);
     return LongFieldWriter.forPrimitive(selector);
   }
 
@@ -110,7 +112,8 @@ public class FieldWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.FLOAT);
     return FloatFieldWriter.forPrimitive(selector);
   }
 
@@ -119,7 +122,8 @@ public class FieldWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.DOUBLE);
     return DoubleFieldWriter.forPrimitive(selector);
   }
 
@@ -139,7 +143,8 @@ public class FieldWriters
       final boolean removeNullBytes
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.STRING_ARRAY);
     return new StringArrayFieldWriter(selector, removeNullBytes);
   }
 
@@ -148,7 +153,8 @@ public class FieldWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.LONG_ARRAY);
     return NumericArrayFieldWriter.getLongArrayFieldWriter(selector);
   }
 
@@ -157,7 +163,8 @@ public class FieldWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.FLOAT_ARRAY);
     return NumericArrayFieldWriter.getFloatArrayFieldWriter(selector);
   }
 
@@ -166,7 +173,8 @@ public class FieldWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.DOUBLE_ARRAY);
     return NumericArrayFieldWriter.getDoubleArrayFieldWriter(selector);
   }
 
@@ -185,7 +193,8 @@ public class FieldWriters
       throw new ISE("No serde for complexTypeName[%s], cannot write column [%s]", columnTypeName, columnName);
     }
 
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.ofComplex(columnTypeName));
     return new ComplexFieldWriter(serde, selector);
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java
index f15361d47ea..f1bcb2e5c70 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java
@@ -39,7 +39,7 @@ import javax.annotation.Nullable;
  *
  * @param <ElementType> Type of the individual array elements
  */
-public abstract class NumericArrayFieldSelector<ElementType extends Number> implements ColumnValueSelector
+public abstract class NumericArrayFieldSelector<ElementType extends Number> implements ColumnValueSelector<Object[]>
 {
   /**
    * Memory containing the serialized values of the array
@@ -81,15 +81,15 @@ public abstract class NumericArrayFieldSelector<ElementType extends Number> impl
 
   @Nullable
   @Override
-  public Object getObject()
+  public Object[] getObject()
   {
     return computeCurrentArray();
   }
 
   @Override
-  public Class classOfObject()
+  public Class<Object[]> classOfObject()
   {
-    return Object.class;
+    return Object[].class;
   }
 
   @Override
@@ -131,7 +131,7 @@ public abstract class NumericArrayFieldSelector<ElementType extends Number> impl
   public abstract int getIndividualFieldSize();
 
   @Nullable
-  private Number[] computeCurrentArray()
+  private Object[] computeCurrentArray()
   {
     final long fieldPosition = fieldPointer.position();
     final long fieldLength = fieldPointer.length();
diff --git a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldWriter.java b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldWriter.java
index e220d6bdc51..8096e323dc3 100644
--- a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldWriter.java
+++ b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldWriter.java
@@ -21,12 +21,10 @@ package org.apache.druid.frame.field;
 
 import org.apache.datasketches.memory.WritableMemory;
 import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.frame.write.FrameWriterUtils;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.ColumnValueSelector;
 
 import javax.annotation.Nullable;
-import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
@@ -128,7 +126,7 @@ public class NumericArrayFieldWriter implements FieldWriter
   @Override
   public long writeTo(WritableMemory memory, long position, long maxSize)
   {
-    Object row = selector.getObject();
+    final Object[] row = (Object[]) selector.getObject();
     if (row == null) {
       int requiredSize = Byte.BYTES;
       if (requiredSize > maxSize) {
@@ -137,18 +135,6 @@ public class NumericArrayFieldWriter implements FieldWriter
       memory.putByte(position, NULL_ROW);
       return requiredSize;
     } else {
-
-      List<? extends Number> list = FrameWriterUtils.getNumericArrayFromObject(row);
-
-      if (list == null) {
-        int requiredSize = Byte.BYTES;
-        if (requiredSize > maxSize) {
-          return -1;
-        }
-        memory.putByte(position, NULL_ROW);
-        return requiredSize;
-      }
-
       // Create a columnValueSelector to write the individual elements re-using the NumericFieldWriter
       AtomicInteger index = new AtomicInteger(0);
       ColumnValueSelector<Number> columnValueSelector = new ColumnValueSelector<Number>()
@@ -199,7 +185,7 @@ public class NumericArrayFieldWriter implements FieldWriter
         @Override
         public Number getObject()
         {
-          return list.get(index.get());
+          return (Number) row[index.get()];
         }
 
         @Override
@@ -215,7 +201,7 @@ public class NumericArrayFieldWriter implements FieldWriter
       // Next [(1 + Numeric Size) x Number of elements of array] bytes are reserved for the elements of the array and
       //  their null markers
       // Last byte is reserved for array termination
-      int requiredSize = Byte.BYTES + (writer.getNumericSizeBytes() + Byte.BYTES) * list.size() + Byte.BYTES;
+      int requiredSize = Byte.BYTES + (writer.getNumericSizeBytes() + Byte.BYTES) * row.length + Byte.BYTES;
 
       if (requiredSize > maxSize) {
         return -1;
@@ -225,7 +211,7 @@ public class NumericArrayFieldWriter implements FieldWriter
       memory.putByte(position + offset, NON_NULL_ROW);
       offset += Byte.BYTES;
 
-      for (; index.get() < list.size(); index.incrementAndGet()) {
+      for (; index.get() < row.length; index.incrementAndGet()) {
         writer.writeTo(
             memory,
             position + offset,
diff --git a/processing/src/main/java/org/apache/druid/frame/write/FrameWriterUtils.java b/processing/src/main/java/org/apache/druid/frame/write/FrameWriterUtils.java
index d857cf33e03..9ba2b29fe51 100644
--- a/processing/src/main/java/org/apache/druid/frame/write/FrameWriterUtils.java
+++ b/processing/src/main/java/org/apache/druid/frame/write/FrameWriterUtils.java
@@ -144,60 +144,16 @@ public class FrameWriterUtils
       @SuppressWarnings("rawtypes") final BaseObjectColumnValueSelector selector
   )
   {
-    Object row = selector.getObject();
+    final Object[] row = (Object[]) selector.getObject();
     if (row == null) {
       return null;
-    } else if (row instanceof String) {
-      return Collections.singletonList(getUtf8ByteBufferFromString((String) row));
-    }
-
-    final List<ByteBuffer> retVal = new ArrayList<>();
-    if (row instanceof List) {
-      for (int i = 0; i < ((List<?>) row).size(); i++) {
-        retVal.add(getUtf8ByteBufferFromString(((List<String>) row).get(i)));
-      }
-    } else if (row instanceof Object[]) {
-      for (Object value : (Object[]) row) {
+    } else {
+      final List<ByteBuffer> retVal = new ArrayList<>();
+      for (Object value : row) {
         retVal.add(getUtf8ByteBufferFromString((String) value));
       }
-    } else {
-      throw new ISE("Unexpected type %s found", row.getClass().getName());
+      return retVal;
     }
-    return retVal;
-  }
-
-  /**
-   * Retrieves a numeric list from a Java object, given that the object is an instance of something that can be returned
-   * from {@link ColumnValueSelector#getObject()} of valid numeric array selectors representations
-   *
-   * While {@link BaseObjectColumnValueSelector} specifies that only instances of {@code Object[]} can be returned from
-   * the numeric array selectors, this method also handles a few more cases which can be encountered if the selector is
-   * directly implemented on top of the group by stuff
-   */
-  @Nullable
-  public static List<? extends Number> getNumericArrayFromObject(Object row)
-  {
-    if (row == null) {
-      return null;
-    } else if (row instanceof Number) {
-      return Collections.singletonList((Number) row);
-    }
-
-    final List<Number> retVal = new ArrayList<>();
-
-    if (row instanceof List) {
-      for (int i = 0; i < ((List<?>) row).size(); i++) {
-        retVal.add((Number) ((List<?>) row).get(i));
-      }
-    } else if (row instanceof Object[]) {
-      for (Object value : (Object[]) row) {
-        retVal.add((Number) value);
-      }
-    } else {
-      throw new ISE("Unexpected type %s found", row.getClass().getName());
-    }
-
-    return retVal;
   }
 
   /**
@@ -275,6 +231,7 @@ public class FrameWriterUtils
    * Whenever "allowNullBytes" is true, "removeNullBytes" must be false. Use the methods {@link #copyByteBufferToMemoryAllowingNullBytes}
    * and {@link #copyByteBufferToMemoryDisallowingNullBytes} to copy between the memory
    * <p>
+   *
    * @throws InvalidNullByteException if "allowNullBytes" and "removeNullBytes" is false and a null byte is encountered
    */
   private static void copyByteBufferToMemory(
diff --git a/processing/src/main/java/org/apache/druid/frame/write/cast/ObjectToArrayColumnValueSelector.java b/processing/src/main/java/org/apache/druid/frame/write/cast/ObjectToArrayColumnValueSelector.java
new file mode 100644
index 00000000000..30537a7ecd9
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/frame/write/cast/ObjectToArrayColumnValueSelector.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.write.cast;
+
+import org.apache.druid.error.DruidException;
+import org.apache.druid.math.expr.ExprEval;
+import org.apache.druid.math.expr.ExpressionType;
+import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.RowIdSupplier;
+
+import javax.annotation.Nullable;
+
+/**
+ * Wraps a {@link ColumnValueSelector}, calls {@link ColumnValueSelector#getObject()}, interprets that value using
+ * {@link ExprEval#ofType}, and casts it using {@link ExprEval#castTo}.
+ */
+public class ObjectToArrayColumnValueSelector implements ColumnValueSelector<Object[]>
+{
+  private final ColumnValueSelector<?> selector;
+  @Nullable
+  private final ExpressionType desiredType;
+  @Nullable
+  private final RowIdSupplier rowIdSupplier;
+
+  public ObjectToArrayColumnValueSelector(
+      final ColumnValueSelector<?> selector,
+      final ExpressionType desiredType,
+      @Nullable final RowIdSupplier rowIdSupplier
+  )
+  {
+    this.selector = selector;
+    this.desiredType = desiredType;
+    this.rowIdSupplier = rowIdSupplier;
+
+    if (!desiredType.isArray() || desiredType.getElementType() == null) {
+      throw DruidException.defensive("Expected array with nonnull element type, got[%s]", desiredType);
+    }
+  }
+
+  @Override
+  public double getDouble()
+  {
+    throw DruidException.defensive("Unexpected call to getDouble on array selector");
+  }
+
+  @Override
+  public float getFloat()
+  {
+    throw DruidException.defensive("Unexpected call to getFloat on array selector");
+  }
+
+  @Override
+  public long getLong()
+  {
+    throw DruidException.defensive("Unexpected call to getLong on array selector");
+  }
+
+  @Override
+  public boolean isNull()
+  {
+    throw DruidException.defensive("Unexpected call to isNull on array selector");
+  }
+
+  @Nullable
+  @Override
+  public Object[] getObject()
+  {
+    return (Object[]) TypeCastSelectors.bestEffortCoerce(selector.getObject(), desiredType);
+  }
+
+  @Override
+  public Class<Object[]> classOfObject()
+  {
+    return Object[].class;
+  }
+
+  @Override
+  public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+  {
+    inspector.visit("selector", selector);
+    inspector.visit("rowIdSupplier", rowIdSupplier);
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/frame/write/cast/ObjectToNumberColumnValueSelector.java b/processing/src/main/java/org/apache/druid/frame/write/cast/ObjectToNumberColumnValueSelector.java
new file mode 100644
index 00000000000..ae09cfd0f20
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/frame/write/cast/ObjectToNumberColumnValueSelector.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.write.cast;
+
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.math.expr.ExpressionType;
+import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.RowIdSupplier;
+
+import javax.annotation.Nullable;
+
+/**
+ * Wraps a {@link ColumnValueSelector}, calls {@link ColumnValueSelector#getObject()} and provides primitive numeric
+ * accessors based on that object value.
+ */
+public class ObjectToNumberColumnValueSelector implements ColumnValueSelector<Number>
+{
+  private final ColumnValueSelector<?> selector;
+  private final ExpressionType desiredType;
+
+  @Nullable
+  private final RowIdSupplier rowIdSupplier;
+
+  @Nullable
+  private Number currentValue;
+  private long currentRowId = RowIdSupplier.INIT;
+
+  /**
+   * Package-private; create using {@link TypeCastSelectors#makeColumnValueSelector} or
+   * {@link TypeCastSelectors#wrapColumnValueSelectorIfNeeded}.
+   */
+  ObjectToNumberColumnValueSelector(
+      final ColumnValueSelector<?> selector,
+      final ExpressionType desiredType,
+      @Nullable final RowIdSupplier rowIdSupplier
+  )
+  {
+    this.selector = selector;
+    this.desiredType = desiredType;
+    this.rowIdSupplier = rowIdSupplier;
+
+    if (!desiredType.isNumeric()) {
+      throw DruidException.defensive("Expected numeric type, got[%s]", desiredType);
+    }
+  }
+
+  @Override
+  public double getDouble()
+  {
+    final Number n = computeIfNeeded();
+    return n == null ? NullHandling.ZERO_DOUBLE : n.doubleValue();
+  }
+
+  @Override
+  public float getFloat()
+  {
+    final Number n = computeIfNeeded();
+    return n == null ? NullHandling.ZERO_FLOAT : n.floatValue();
+  }
+
+  @Override
+  public long getLong()
+  {
+    final Number n = computeIfNeeded();
+    return n == null ? NullHandling.ZERO_LONG : n.longValue();
+  }
+
+  @Override
+  public boolean isNull()
+  {
+    return computeIfNeeded() == null;
+  }
+
+  @Nullable
+  @Override
+  public Number getObject()
+  {
+    return computeIfNeeded();
+  }
+
+  @Override
+  public Class<Number> classOfObject()
+  {
+    return Number.class;
+  }
+
+  @Override
+  public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+  {
+    inspector.visit("selector", selector);
+    inspector.visit("rowIdSupplier", rowIdSupplier);
+  }
+
+  @Nullable
+  private Number computeIfNeeded()
+  {
+    if (rowIdSupplier == null) {
+      return eval();
+    } else {
+      final long rowId = rowIdSupplier.getRowId();
+
+      if (currentRowId != rowId) {
+        currentValue = eval();
+        currentRowId = rowId;
+      }
+
+      return currentValue;
+    }
+  }
+
+  @Nullable
+  private Number eval()
+  {
+    return (Number) TypeCastSelectors.bestEffortCoerce(selector.getObject(), desiredType);
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/frame/write/cast/TypeCastSelectors.java b/processing/src/main/java/org/apache/druid/frame/write/cast/TypeCastSelectors.java
new file mode 100644
index 00000000000..ec6f16aad75
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/frame/write/cast/TypeCastSelectors.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.write.cast;
+
+import org.apache.druid.error.DruidException;
+import org.apache.druid.math.expr.ExprEval;
+import org.apache.druid.math.expr.ExpressionType;
+import org.apache.druid.segment.BaseObjectColumnValueSelector;
+import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.RowIdSupplier;
+import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.ValueType;
+
+import javax.annotation.Nullable;
+
+public class TypeCastSelectors
+{
+  /**
+   * Create a {@link ColumnValueSelector} that does its own typecasting if necessary. If typecasting is not necessary,
+   * returns a selector directly from the underlying {@link ColumnSelectorFactory}.
+   *
+   * @param columnSelectorFactory underlying factory
+   * @param column                column name
+   * @param desiredType           desired type of selector. Can be anything except {@link ColumnType#STRING}.
+   *                              For strings, use {@link DimensionSelector} rather than {@link ColumnValueSelector}.
+   */
+  public static ColumnValueSelector<?> makeColumnValueSelector(
+      final ColumnSelectorFactory columnSelectorFactory,
+      final String column,
+      final ColumnType desiredType
+  )
+  {
+    final ColumnValueSelector<?> selector = columnSelectorFactory.makeColumnValueSelector(column);
+    final ColumnCapabilities selectorCapabilities = columnSelectorFactory.getColumnCapabilities(column);
+
+    return wrapColumnValueSelectorIfNeeded(
+        selector,
+        selectorCapabilities,
+        columnSelectorFactory.getRowIdSupplier(),
+        desiredType
+    );
+  }
+
+  /**
+   * Wraps a {@link ColumnValueSelector} with a type casting selector if necessary. If typecasting is not necessary,
+   * returns the original selector.
+   *
+   * @param selector             selector
+   * @param selectorCapabilities capabilities for the selector, from {@link ColumnSelectorFactory#getColumnCapabilities}
+   * @param rowIdSupplier        row id supplier, from {@link ColumnSelectorFactory#getRowIdSupplier()}
+   * @param desiredType          desired type for the returned selector
+   */
+  public static ColumnValueSelector<?> wrapColumnValueSelectorIfNeeded(
+      final ColumnValueSelector<?> selector,
+      @Nullable final ColumnCapabilities selectorCapabilities,
+      @Nullable final RowIdSupplier rowIdSupplier,
+      final ColumnType desiredType
+  )
+  {
+    final ExpressionType desiredExpressionType = ExpressionType.fromColumnType(desiredType);
+
+    if (desiredType.is(ValueType.STRING)) {
+      throw DruidException.defensive("Type[%s] should be read using a DimensionSelector", desiredType);
+    } else if (desiredType.isNumeric()
+               && (selectorCapabilities == null || !selectorCapabilities.isNumeric())) {
+      // When capabilities are unknown, or known to be non-numeric, fall back to getObject() and explicit typecasting.
+      // This avoids using primitive numeric accessors (getLong / getDouble / getFloat / isNull) on a selector that
+      // may not support them.
+      return new ObjectToNumberColumnValueSelector(selector, desiredExpressionType, rowIdSupplier);
+    } else if (desiredType.isArray()) {
+      // Always wrap if desiredType is an array. Even if the underlying selector claims to offer the same type as
+      // desiredType, it may fail to respect the BaseObjectColumnValueSelector contract. For example, it may return
+      // List rather than Object[]. (RowBasedColumnSelectorFactory can do this if used incorrectly, i.e., if the
+      // ColumnInspector declares type ARRAY<X> for a column, but the RowAdapter does not provide Object[].)
+      return new ObjectToArrayColumnValueSelector(selector, desiredExpressionType, rowIdSupplier);
+    } else {
+      // OK to return the original selector.
+      return selector;
+    }
+  }
+
+  /**
+   * Coerce an object to an object compatible with what {@link BaseObjectColumnValueSelector#getObject()} for a column
+   * of the provided desiredType.
+   *
+   * @param obj         object
+   * @param desiredType desired type
+   */
+  @Nullable
+  public static Object bestEffortCoerce(
+      @Nullable final Object obj,
+      @Nullable final ExpressionType desiredType
+  )
+  {
+    if (obj == null || desiredType == null) {
+      return obj;
+    }
+
+    return ExprEval.ofType(desiredType, obj).value();
+  }
+}
diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java
index 93f0c12bae6..6efe1870816 100644
--- a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java
+++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java
@@ -21,6 +21,7 @@ package org.apache.druid.frame.write.columnar;
 
 import org.apache.druid.frame.allocation.MemoryAllocator;
 import org.apache.druid.frame.write.UnsupportedColumnTypeException;
+import org.apache.druid.frame.write.cast.TypeCastSelectors;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.segment.ColumnSelectorFactory;
@@ -102,7 +103,8 @@ public class FrameColumnWriters
   )
   {
     final ColumnCapabilities capabilities = selectorFactory.getColumnCapabilities(columnName);
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.LONG);
     return new LongFrameColumnWriter(selector, allocator, hasNullsForNumericWriter(capabilities));
   }
 
@@ -113,7 +115,8 @@ public class FrameColumnWriters
   )
   {
     final ColumnCapabilities capabilities = selectorFactory.getColumnCapabilities(columnName);
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.FLOAT);
     return new FloatFrameColumnWriter(selector, allocator, hasNullsForNumericWriter(capabilities));
   }
 
@@ -124,7 +127,8 @@ public class FrameColumnWriters
   )
   {
     final ColumnCapabilities capabilities = selectorFactory.getColumnCapabilities(columnName);
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.DOUBLE);
     return new DoubleFrameColumnWriter(selector, allocator, hasNullsForNumericWriter(capabilities));
   }
 
@@ -149,7 +153,8 @@ public class FrameColumnWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.STRING_ARRAY);
     return new StringArrayFrameColumnWriterImpl(selector, allocator);
   }
 
@@ -159,7 +164,8 @@ public class FrameColumnWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.LONG_ARRAY);
     return new LongArrayFrameColumnWriter(selector, allocator);
   }
 
@@ -169,7 +175,8 @@ public class FrameColumnWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.FLOAT_ARRAY);
     return new FloatArrayFrameColumnWriter(selector, allocator);
   }
 
@@ -179,7 +186,8 @@ public class FrameColumnWriters
       final String columnName
   )
   {
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.DOUBLE_ARRAY);
     return new DoubleArrayFrameColumnWriter(selector, allocator);
   }
 
@@ -199,7 +207,8 @@ public class FrameColumnWriters
       throw new ISE("No serde for complexTypeName[%s], cannot write column [%s]", columnTypeName, columnName);
     }
 
-    final ColumnValueSelector<?> selector = selectorFactory.makeColumnValueSelector(columnName);
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(selectorFactory, columnName, ColumnType.ofComplex(columnTypeName));
     return new ComplexFrameColumnWriter(selector, allocator, serde);
   }
 
diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java
index 619bf53b8d3..432ce8b97aa 100644
--- a/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java
+++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java
@@ -24,11 +24,8 @@ import org.apache.druid.error.DruidException;
 import org.apache.druid.frame.allocation.AppendableMemory;
 import org.apache.druid.frame.allocation.MemoryAllocator;
 import org.apache.druid.frame.allocation.MemoryRange;
-import org.apache.druid.frame.write.FrameWriterUtils;
 import org.apache.druid.segment.ColumnValueSelector;
 
-import java.util.List;
-
 /**
  * Parent class for the family of writers writing numeric arrays in columnar frames. Since the numeric primitives are
  * fixed width, we don't need to store the width of each element. The memory layout of a column written by this writer
@@ -119,8 +116,8 @@ public abstract class NumericArrayFrameColumnWriter implements FrameColumnWriter
   @Override
   public boolean addSelection()
   {
-    List<? extends Number> numericArray = FrameWriterUtils.getNumericArrayFromObject(selector.getObject());
-    int rowLength = numericArray == null ? 0 : numericArray.size();
+    final Object[] row = (Object[]) selector.getObject();
+    int rowLength = row == null ? 0 : row.length;
 
     // Begin memory allocations before writing
     if ((long) lastCumulativeRowLength + rowLength > Integer.MAX_VALUE) {
@@ -142,7 +139,7 @@ public abstract class NumericArrayFrameColumnWriter implements FrameColumnWriter
 
     final MemoryRange<WritableMemory> rowLengthsCursor = cumulativeRowLengths.cursor();
 
-    if (numericArray == null) {
+    if (row == null) {
       rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), -(lastCumulativeRowLength + rowLength) - 1);
     } else {
       rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), lastCumulativeRowLength + rowLength);
@@ -155,7 +152,7 @@ public abstract class NumericArrayFrameColumnWriter implements FrameColumnWriter
     final MemoryRange<WritableMemory> rowDataCursor = rowLength > 0 ? rowData.cursor() : null;
 
     for (int i = 0; i < rowLength; ++i) {
-      final Number element = numericArray.get(i);
+      final Number element = (Number) row[i];
       final long memoryOffset = rowDataCursor.start() + ((long) elementSizeBytes() * i);
       if (element == null) {
         rowNullityDataCursor.memory()
diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
index 4bde1519221..a16b563c77c 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
@@ -406,7 +406,7 @@ public abstract class ExprEval<T>
   public static ExprEval bestEffortOf(@Nullable Object val)
   {
     if (val == null) {
-      return new StringExprEval(null);
+      return StringExprEval.OF_NULL;
     }
     if (val instanceof ExprEval) {
       return (ExprEval) val;
@@ -519,6 +519,12 @@ public abstract class ExprEval<T>
     return ofComplex(ExpressionType.UNKNOWN_COMPLEX, val);
   }
 
+  /**
+   * Create an eval of the provided type. Coerces the provided object to the desired type.
+   *
+   * @param type  type, or null to be equivalent to {@link #bestEffortOf(Object)}
+   * @param value object to be coerced to the type
+   */
   public static ExprEval ofType(@Nullable ExpressionType type, @Nullable Object value)
   {
     if (type == null) {
@@ -1145,30 +1151,31 @@ public abstract class ExprEval<T>
       switch (castTo.getType()) {
         case DOUBLE:
           return ExprEval.ofDouble(computeNumber());
+
         case LONG:
           return ExprEval.ofLong(computeNumber());
+
         case STRING:
           return this;
+
         case ARRAY:
           if (value == null) {
             return new ArrayExprEval(castTo, null);
           }
-          final Number number = computeNumber();
-          switch (castTo.getElementType().getType()) {
-            case DOUBLE:
-              return ExprEval.ofDoubleArray(
-                  new Object[]{number == null ? null : number.doubleValue()}
-              );
-            case LONG:
-              return ExprEval.ofLongArray(
-                  new Object[]{number == null ? null : number.longValue()}
-              );
-            case STRING:
-              return ExprEval.ofStringArray(new Object[]{value});
-            default:
-              ExpressionType elementType = (ExpressionType) castTo.getElementType();
-              return new ArrayExprEval(castTo, new Object[]{castTo(elementType).value()});
+          ExprType type = castTo.getElementType().getType();
+          if (type == ExprType.DOUBLE) {
+            final Number number = computeNumber();
+            return ExprEval.ofDoubleArray(new Object[]{number == null ? null : number.doubleValue()});
+          } else if (type == ExprType.LONG) {
+            final Number number = computeNumber();
+            return ExprEval.ofLongArray(new Object[]{number == null ? null : number.longValue()});
+          } else if (type == ExprType.STRING) {
+            return ExprEval.ofStringArray(new Object[]{value});
           }
+
+          ExpressionType elementType = (ExpressionType) castTo.getElementType();
+          return new ArrayExprEval(castTo, new Object[]{castTo(elementType).value()});
+
         case COMPLEX:
           if (ExpressionType.NESTED_DATA.equals(castTo)) {
             return new NestedDataExprEval(value);
diff --git a/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java b/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java
index 6aba25ddf22..44d8ed5a26d 100644
--- a/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java
@@ -50,7 +50,7 @@ public class StringArrayFieldWriterTest extends InitializedNullHandlingTest
   public MockitoRule mockitoRule = MockitoJUnit.rule().strictness(Strictness.STRICT_STUBS);
 
   @Mock
-  public BaseObjectColumnValueSelector<List<String>> selector;
+  public BaseObjectColumnValueSelector<Object[]> selector;
 
   private WritableMemory memory;
   private FieldWriter fieldWriter;
@@ -115,7 +115,8 @@ public class StringArrayFieldWriterTest extends InitializedNullHandlingTest
 
   private void mockSelector(@Nullable final List<String> values)
   {
-    Mockito.when(selector.getObject()).thenReturn(values);
+    final Object[] arr = values == null ? null : values.toArray();
+    Mockito.when(selector.getObject()).thenReturn(arr);
   }
 
   private long writeToMemory(final FieldWriter writer)
diff --git a/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java
index b0f589ed480..200c469269f 100644
--- a/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java
+++ b/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java
@@ -60,7 +60,7 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
   public MockitoRule mockitoRule = MockitoJUnit.rule().strictness(Strictness.STRICT_STUBS);
 
   @Mock
-  public BaseObjectColumnValueSelector<List<String>> writeSelector;
+  public BaseObjectColumnValueSelector<Object[]> writeSelector;
 
   private WritableMemory memory;
   private FieldWriter fieldWriter;
@@ -277,7 +277,8 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
 
   private void writeToMemory(@Nullable final List<String> values)
   {
-    Mockito.when(writeSelector.getObject()).thenReturn(values);
+    final Object[] arr = values == null ? null : values.toArray();
+    Mockito.when(writeSelector.getObject()).thenReturn(arr);
 
     if (fieldWriter.writeTo(memory, MEMORY_POSITION, memory.getCapacity() - MEMORY_POSITION) < 0) {
       throw new ISE("Could not write");
diff --git a/processing/src/test/java/org/apache/druid/frame/write/cast/TypeCastSelectorsTest.java b/processing/src/test/java/org/apache/druid/frame/write/cast/TypeCastSelectorsTest.java
new file mode 100644
index 00000000000..dc2a4a5b46c
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/frame/write/cast/TypeCastSelectorsTest.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.frame.write.cast;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.query.dimension.DimensionSpec;
+import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.util.Map;
+
+public class TypeCastSelectorsTest extends InitializedNullHandlingTest
+{
+  private final ColumnSelectorFactory testColumnSelectorFactory = new TestColumnSelectorFactory(
+      RowSignature.builder()
+                  .add("x", ColumnType.STRING)
+                  .add("y", ColumnType.STRING)
+                  .add("z", ColumnType.STRING)
+                  .add("da", ColumnType.DOUBLE_ARRAY)
+                  .build(),
+      ImmutableMap.<String, Object>builder()
+                  .put("x", "12.3")
+                  .put("y", "abc")
+                  .put("da", new Object[]{1.2d, 2.3d})
+                  .build() // z is null
+  );
+
+  @Test
+  public void test_readXAsLong()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "x", ColumnType.LONG);
+
+    Assert.assertEquals(12L, selector.getLong());
+    Assert.assertEquals(12d, selector.getDouble(), 0);
+    Assert.assertEquals(12f, selector.getFloat(), 0);
+    Assert.assertFalse(selector.isNull());
+    Assert.assertEquals(12L, selector.getObject());
+  }
+
+  @Test
+  public void test_readXAsDouble()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "x", ColumnType.DOUBLE);
+
+    Assert.assertEquals(12L, selector.getLong());
+    Assert.assertEquals(12.3d, selector.getDouble(), 0);
+    Assert.assertEquals(12.3f, selector.getFloat(), 0);
+    Assert.assertFalse(selector.isNull());
+    Assert.assertEquals(12.3d, selector.getObject());
+  }
+
+  @Test
+  public void test_readXAsFloat()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "x", ColumnType.FLOAT);
+
+    Assert.assertEquals(12L, selector.getLong());
+    Assert.assertEquals(12.3d, selector.getDouble(), 0.001);
+    Assert.assertEquals(12.3f, selector.getFloat(), 0);
+    Assert.assertFalse(selector.isNull());
+    Assert.assertEquals(12.3d, selector.getObject());
+  }
+
+  @Test
+  public void test_readXAsLongArray()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "x", ColumnType.LONG_ARRAY);
+
+    Assert.assertThrows(DruidException.class, selector::getLong);
+    Assert.assertThrows(DruidException.class, selector::getDouble);
+    Assert.assertThrows(DruidException.class, selector::getFloat);
+    Assert.assertThrows(DruidException.class, selector::isNull);
+    Assert.assertArrayEquals(new Object[]{12L}, (Object[]) selector.getObject());
+  }
+
+  @Test
+  public void test_readXAsStringArray()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "x", ColumnType.STRING_ARRAY);
+
+    Assert.assertThrows(DruidException.class, selector::getLong);
+    Assert.assertThrows(DruidException.class, selector::getDouble);
+    Assert.assertThrows(DruidException.class, selector::getFloat);
+    Assert.assertThrows(DruidException.class, selector::isNull);
+    Assert.assertArrayEquals(new Object[]{"12.3"}, (Object[]) selector.getObject());
+  }
+
+  @Test
+  public void test_readYAsLong()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "y", ColumnType.LONG);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readYAsDouble()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "y", ColumnType.DOUBLE);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readYAsFloat()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "y", ColumnType.FLOAT);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readYAsLongArray()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "y", ColumnType.LONG_ARRAY);
+
+    Assert.assertThrows(DruidException.class, selector::getLong);
+    Assert.assertThrows(DruidException.class, selector::getDouble);
+    Assert.assertThrows(DruidException.class, selector::getFloat);
+    Assert.assertThrows(DruidException.class, selector::isNull);
+    Assert.assertArrayEquals(new Object[]{null}, (Object[]) selector.getObject());
+  }
+
+  @Test
+  public void test_readYAsStringArray()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "y", ColumnType.STRING_ARRAY);
+
+    Assert.assertThrows(DruidException.class, selector::getLong);
+    Assert.assertThrows(DruidException.class, selector::getDouble);
+    Assert.assertThrows(DruidException.class, selector::getFloat);
+    Assert.assertThrows(DruidException.class, selector::isNull);
+    Assert.assertArrayEquals(new Object[]{"abc"}, (Object[]) selector.getObject());
+  }
+
+  @Test
+  public void test_readZAsLong()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "z", ColumnType.LONG);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readZAsDouble()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "z", ColumnType.DOUBLE);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readZAsFloat()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "z", ColumnType.FLOAT);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readZAsLongArray()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "z", ColumnType.LONG_ARRAY);
+
+    Assert.assertThrows(DruidException.class, selector::getLong);
+    Assert.assertThrows(DruidException.class, selector::getDouble);
+    Assert.assertThrows(DruidException.class, selector::getFloat);
+    Assert.assertThrows(DruidException.class, selector::isNull);
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readZAsStringArray()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "z", ColumnType.STRING_ARRAY);
+
+    Assert.assertThrows(DruidException.class, selector::getLong);
+    Assert.assertThrows(DruidException.class, selector::getDouble);
+    Assert.assertThrows(DruidException.class, selector::getFloat);
+    Assert.assertThrows(DruidException.class, selector::isNull);
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readDaAsLong()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "da", ColumnType.LONG);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readDaAsDouble()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "da", ColumnType.DOUBLE);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readDaAsFloat()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "da", ColumnType.FLOAT);
+
+    Assert.assertEquals(0L, selector.getLong());
+    Assert.assertEquals(0d, selector.getDouble(), 0);
+    Assert.assertEquals(0f, selector.getFloat(), 0);
+    Assert.assertTrue(selector.isNull());
+    Assert.assertNull(selector.getObject());
+  }
+
+  @Test
+  public void test_readDaAsLongArray()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "da", ColumnType.LONG_ARRAY);
+
+    Assert.assertThrows(DruidException.class, selector::getLong);
+    Assert.assertThrows(DruidException.class, selector::getDouble);
+    Assert.assertThrows(DruidException.class, selector::getFloat);
+    Assert.assertThrows(DruidException.class, selector::isNull);
+    Assert.assertArrayEquals(new Object[]{1L, 2L}, (Object[]) selector.getObject());
+  }
+
+  @Test
+  public void test_readDaAsStringArray()
+  {
+    final ColumnValueSelector<?> selector =
+        TypeCastSelectors.makeColumnValueSelector(testColumnSelectorFactory, "da", ColumnType.STRING_ARRAY);
+
+    Assert.assertThrows(DruidException.class, selector::getLong);
+    Assert.assertThrows(DruidException.class, selector::getDouble);
+    Assert.assertThrows(DruidException.class, selector::getFloat);
+    Assert.assertThrows(DruidException.class, selector::isNull);
+    Assert.assertArrayEquals(new Object[]{"1.2", "2.3"}, (Object[]) selector.getObject());
+  }
+
+  /**
+   * Implementation that returns a fixed value per column from {@link ColumnValueSelector#getObject()}. Other
+   * methods, such as {@link ColumnValueSelector#getLong()} throw exceptions. This is meant to help validate
+   * that those other methods are *not* called.
+   */
+  private static class TestColumnSelectorFactory implements ColumnSelectorFactory
+  {
+    private final RowSignature signature;
+    private final Map<String, Object> columnValues;
+
+    public TestColumnSelectorFactory(final RowSignature signature, final Map<String, Object> columnValues)
+    {
+      this.signature = signature;
+      this.columnValues = columnValues;
+    }
+
+    @Override
+    public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec)
+    {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ColumnValueSelector<?> makeColumnValueSelector(String columnName)
+    {
+      return new ColumnValueSelector<Object>()
+      {
+        @Override
+        public double getDouble()
+        {
+          throw new UnsupportedOperationException("Should not be called");
+        }
+
+        @Override
+        public float getFloat()
+        {
+          throw new UnsupportedOperationException("Should not be called");
+        }
+
+        @Override
+        public long getLong()
+        {
+          throw new UnsupportedOperationException("Should not be called");
+        }
+
+        @Override
+        public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+        {
+          throw new UnsupportedOperationException("Should not be called");
+        }
+
+        @Override
+        public boolean isNull()
+        {
+          throw new UnsupportedOperationException("Should not be called");
+        }
+
+        @Nullable
+        @Override
+        public Object getObject()
+        {
+          return columnValues.get(columnName);
+        }
+
+        @Override
+        public Class<?> classOfObject()
+        {
+          throw new UnsupportedOperationException("Should not be called");
+        }
+      };
+    }
+
+    @Nullable
+    @Override
+    public ColumnCapabilities getColumnCapabilities(String column)
+    {
+      return signature.getColumnCapabilities(column);
+    }
+  }
+}

From 476b205efabda70ff57d3cbd4288bf74ab28044b Mon Sep 17 00:00:00 2001
From: Virushade <70288012+GWphua@users.noreply.github.com>
Date: Fri, 6 Sep 2024 11:18:00 +0800
Subject: [PATCH 98/99] Docs: Fix language in Schema Design docs (#17010)

---
 docs/ingestion/schema-design.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/ingestion/schema-design.md b/docs/ingestion/schema-design.md
index ce57b207fea..eeb082d95ec 100644
--- a/docs/ingestion/schema-design.md
+++ b/docs/ingestion/schema-design.md
@@ -57,7 +57,7 @@ In Druid, on the other hand, it is common to use totally flat datasources that d
 the example of the "sales" table, in Druid it would be typical to store "product_id", "product_name", and
 "product_category" as dimensions directly in a Druid "sales" datasource, without using a separate "products" table.
 Totally flat schemas substantially increase performance, since the need for joins is eliminated at query time. As an
-an added speed boost, this also allows Druid's query layer to operate directly on compressed dictionary-encoded data.
+added speed boost, this also allows Druid's query layer to operate directly on compressed dictionary-encoded data.
 Perhaps counter-intuitively, this does _not_ substantially increase storage footprint relative to normalized schemas,
 since Druid uses dictionary encoding to effectively store just a single integer per row for string columns.
 
@@ -101,7 +101,7 @@ see [partitioning and sorting](./partitioning.md) below for details).
 * Create other dimensions for attributes attached to your data points. These are often called "tags" in timeseries
 database systems.
 * Create [metrics](../querying/aggregations.md) corresponding to the types of aggregations that you want to be able
-to query. Typically this includes "sum", "min", and "max" (in one of the long, float, or double flavors). If you want the ability
+to query. Typically, this includes "sum", "min", and "max" (in one of the long, float, or double flavors). If you want the ability
 to compute percentiles or quantiles, use Druid's [approximate aggregators](../querying/aggregations.md#approximate-aggregations).
 * Consider enabling [rollup](./rollup.md), which will allow Druid to potentially combine multiple points into one
 row in your Druid datasource. This can be useful if you want to store data at a different time granularity than it is
@@ -160,7 +160,7 @@ approximate distinct counts, and you'll reduce your storage footprint.
 
 Sketches reduce memory footprint at query time because they limit the amount of data that needs to be shuffled between
 servers. For example, in a quantile computation, instead of needing to send all data points to a central location
-so they can be sorted and the quantile can be computed, Druid instead only needs to send a sketch of the points. This
+so that they can be sorted and the quantile can be computed, Druid instead only needs to send a sketch of the points. This
 can reduce data transfer needs to mere kilobytes.
 
 For details about the sketches available in Druid, see the
@@ -255,7 +255,7 @@ Druid can infer the schema for your data in one of two ways:
 
 You can have Druid infer the schema and types for your data partially or fully by setting `dimensionsSpec.useSchemaDiscovery` to `true` and defining some or no dimensions in the dimensions list. 
 
-When performing type-aware schema discovery, Druid can discover all of the columns of your input data (that aren't in
+When performing type-aware schema discovery, Druid can discover all the columns of your input data (that are not present in
 the exclusion list). Druid automatically chooses the most appropriate native Druid type among `STRING`, `LONG`,
 `DOUBLE`, `ARRAY<STRING>`, `ARRAY<LONG>`, `ARRAY<DOUBLE>`, or `COMPLEX<json>` for nested data. For input formats with
 native boolean types, Druid ingests these values as longs if `druid.expressions.useStrictBooleans` is set to `true`
@@ -298,7 +298,7 @@ If you previously used string-based schema discovery and want to migrate to type
 ### Including the same column as a dimension and a metric
 
 One workflow with unique IDs is to be able to filter on a particular ID, while still being able to do fast unique counts on the ID column.
-If you are not using schema-less dimensions, this use case is supported by setting the `name` of the metric to something different than the dimension.
+If you are not using schema-less dimensions, this use case is supported by setting the `name` of the metric to something different from the dimension.
 If you are using schema-less dimensions, the best practice here is to include the same column twice, once as a dimension, and as a `hyperUnique` metric. This may involve
 some work at ETL time.
 

From 73ff9f9047d69d55bf880e629efeec07ce92e528 Mon Sep 17 00:00:00 2001
From: Adarsh Sanjeev <adarshsanjeev@gmail.com>
Date: Fri, 6 Sep 2024 09:56:35 +0530
Subject: [PATCH 99/99] Convert MSQTerminalStageSpecFactory into an interface
 (#16996)

* Convert MSQTerminalStageSpecFactory into an interface

* Rename class and remove useless constructor
---
 .../apache/druid/msq/guice/MSQSqlModule.java  |  2 +-
 .../guice/MSQTerminalStageSpecFactory.java    | 11 ++----
 ...entGenerationTerminalStageSpecFactory.java | 37 +++++++++++++++++++
 .../druid/msq/sql/MSQTaskSqlEngine.java       |  7 ----
 .../druid/msq/exec/TestMSQSqlModule.java      |  3 +-
 .../apache/druid/msq/test/MSQTestBase.java    |  4 +-
 6 files changed, 45 insertions(+), 19 deletions(-)
 create mode 100644 extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/SegmentGenerationTerminalStageSpecFactory.java

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQSqlModule.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQSqlModule.java
index a523fd6c012..5d837940e19 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQSqlModule.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQSqlModule.java
@@ -54,7 +54,7 @@ public class MSQSqlModule implements DruidModule
     // We want this module to bring InputSourceModule along for the ride.
     binder.install(new InputSourceModule());
 
-    binder.bind(MSQTerminalStageSpecFactory.class).toInstance(new MSQTerminalStageSpecFactory());
+    binder.bind(MSQTerminalStageSpecFactory.class).to(SegmentGenerationTerminalStageSpecFactory.class).in(LazySingleton.class);
 
     binder.bind(MSQTaskSqlEngine.class).in(LazySingleton.class);
 
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQTerminalStageSpecFactory.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQTerminalStageSpecFactory.java
index d347322a7fb..d5a1bd79455 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQTerminalStageSpecFactory.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQTerminalStageSpecFactory.java
@@ -19,19 +19,14 @@
 
 package org.apache.druid.msq.guice;
 
-import org.apache.druid.msq.indexing.destination.SegmentGenerationStageSpec;
 import org.apache.druid.msq.indexing.destination.TerminalStageSpec;
 import org.apache.druid.sql.calcite.planner.PlannerContext;
 import org.apache.druid.sql.calcite.rel.DruidQuery;
 
-public class MSQTerminalStageSpecFactory
+public interface MSQTerminalStageSpecFactory
 {
   /**
-   * Creates a {@link TerminalStageSpec} which determines the final of a query. Currently, always returns a segment
-   * generation spec, but this can be used to configure a wide range of behaviours.
+   * Creates a {@link TerminalStageSpec} which determines the final of a query.
    */
-  public TerminalStageSpec createTerminalStageSpec(DruidQuery druidQuery, PlannerContext plannerContext)
-  {
-    return SegmentGenerationStageSpec.instance();
-  }
+  TerminalStageSpec createTerminalStageSpec(DruidQuery druidQuery, PlannerContext plannerContext);
 }
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/SegmentGenerationTerminalStageSpecFactory.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/SegmentGenerationTerminalStageSpecFactory.java
new file mode 100644
index 00000000000..f761c0616a6
--- /dev/null
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/SegmentGenerationTerminalStageSpecFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.guice;
+
+import org.apache.druid.msq.indexing.destination.SegmentGenerationStageSpec;
+import org.apache.druid.msq.indexing.destination.TerminalStageSpec;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.rel.DruidQuery;
+
+/**
+ * Configures ingestion queries to create new segments with the results in all cases.
+ */
+public class SegmentGenerationTerminalStageSpecFactory implements MSQTerminalStageSpecFactory
+{
+  @Override
+  public TerminalStageSpec createTerminalStageSpec(DruidQuery druidQuery, PlannerContext plannerContext)
+  {
+    return SegmentGenerationStageSpec.instance();
+  }
+}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
index 01d87805f78..9e07a909f4e 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskSqlEngine.java
@@ -20,7 +20,6 @@
 package org.apache.druid.msq.sql;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Inject;
@@ -90,12 +89,6 @@ public class MSQTaskSqlEngine implements SqlEngine
   private final MSQTerminalStageSpecFactory terminalStageSpecFactory;
 
   @Inject
-  public MSQTaskSqlEngine(final OverlordClient overlordClient, final ObjectMapper jsonMapper)
-  {
-    this(overlordClient, jsonMapper, new MSQTerminalStageSpecFactory());
-  }
-
-  @VisibleForTesting
   public MSQTaskSqlEngine(
       final OverlordClient overlordClient,
       final ObjectMapper jsonMapper,
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/TestMSQSqlModule.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/TestMSQSqlModule.java
index e5a0f1cb06e..5bf4bbd44dc 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/TestMSQSqlModule.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/TestMSQSqlModule.java
@@ -26,6 +26,7 @@ import com.google.inject.Provides;
 import org.apache.druid.guice.LazySingleton;
 import org.apache.druid.initialization.ServerInjectorBuilderTest.TestDruidModule;
 import org.apache.druid.msq.guice.MultiStageQuery;
+import org.apache.druid.msq.guice.SegmentGenerationTerminalStageSpecFactory;
 import org.apache.druid.msq.sql.MSQTaskSqlEngine;
 import org.apache.druid.msq.test.MSQTestBase;
 import org.apache.druid.msq.test.MSQTestOverlordServiceClient;
@@ -53,7 +54,7 @@ public class TestMSQSqlModule extends TestDruidModule
       ObjectMapper queryJsonMapper,
       MSQTestOverlordServiceClient indexingServiceClient)
   {
-    return new MSQTaskSqlEngine(indexingServiceClient, queryJsonMapper);
+    return new MSQTaskSqlEngine(indexingServiceClient, queryJsonMapper, new SegmentGenerationTerminalStageSpecFactory());
   }
 
   @Provides
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
index fe725528f40..bb8691b9322 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
@@ -91,8 +91,8 @@ import org.apache.druid.msq.guice.MSQDurableStorageModule;
 import org.apache.druid.msq.guice.MSQExternalDataSourceModule;
 import org.apache.druid.msq.guice.MSQIndexingModule;
 import org.apache.druid.msq.guice.MSQSqlModule;
-import org.apache.druid.msq.guice.MSQTerminalStageSpecFactory;
 import org.apache.druid.msq.guice.MultiStageQuery;
+import org.apache.druid.msq.guice.SegmentGenerationTerminalStageSpecFactory;
 import org.apache.druid.msq.indexing.InputChannelFactory;
 import org.apache.druid.msq.indexing.MSQControllerTask;
 import org.apache.druid.msq.indexing.MSQSpec;
@@ -551,7 +551,7 @@ public class MSQTestBase extends BaseCalciteQueryTest
     final SqlEngine engine = new MSQTaskSqlEngine(
         indexingServiceClient,
         qf.queryJsonMapper().copy().registerModules(new MSQSqlModule().getJacksonModules()),
-        new MSQTerminalStageSpecFactory()
+        new SegmentGenerationTerminalStageSpecFactory()
     );
 
     PlannerFactory plannerFactory = new PlannerFactory(