From 41e51b21c3396a78c7a6b736bcb87903876991b2 Mon Sep 17 00:00:00 2001
From: AmatyaAvadhanula <amatya.avadhanula@imply.io>
Date: Wed, 5 Oct 2022 05:35:17 +0530
Subject: [PATCH] Make http options the default configurations (#13092)

Druid currently uses Zookeeper dependent options as the default.
This commit updates the following to use HTTP as the default instead.
- task runner. `druid.indexer.runner.type=remote -> httpRemote`
- load queue peon. `druid.coordinator.loadqueuepeon.type=curator -> http`
- server inventory view. `druid.serverview.type=curator -> http`
---
 dev/intellij-setup.md                                     | 4 ++--
 docs/configuration/index.md                               | 8 ++++----
 .../master/coordinator-overlord/runtime.properties        | 1 -
 .../large/coordinator-overlord/runtime.properties         | 1 -
 .../medium/coordinator-overlord/runtime.properties        | 1 -
 .../coordinator-overlord/runtime.properties               | 1 -
 .../coordinator-overlord/runtime.properties               | 1 -
 .../small/coordinator-overlord/runtime.properties         | 1 -
 .../xlarge/coordinator-overlord/runtime.properties        | 1 -
 .../druid/client/FilteredServerInventoryViewProvider.java | 2 +-
 .../apache/druid/client/ServerInventoryViewProvider.java  | 2 +-
 .../druid/server/coordinator/DruidCoordinatorConfig.java  | 2 +-
 .../server/coordinator/DruidCoordinatorConfigTest.java    | 1 +
 .../src/main/java/org/apache/druid/cli/CliOverlord.java   | 2 +-
 14 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/dev/intellij-setup.md b/dev/intellij-setup.md
index 2ddceb90978..5c4cdc52f3d 100644
--- a/dev/intellij-setup.md
+++ b/dev/intellij-setup.md
@@ -93,7 +93,7 @@ You can configure application definitions in XML for import into IntelliJ. Below
   <configuration default="false" name="Historical" type="Application" factoryName="Application">
     <extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
     <option name="MAIN_CLASS_NAME" value="org.apache.druid.cli.Main" />
-    <option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx2G -XX:MaxJavaStackTraceDepth=9999 -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Dlog4j.configurationFile=$PROJECT_DIR$/core/src/main/resources/log4j2.debug.xml -Ddruid.host=localhost -Ddruid.service=historical -Ddruid.processing.buffer.sizeBytes=100000000 -Ddruid.extensions.hadoopDependenciesDir=$PROJECT_DIR$/distribution/target/hadoop-dependencies/ -Ddruid.extensions.directory=$PROJECT_DIR$/distribution/target/extensions/ -Ddruid.extensions.loadList=[\&quot;druid-s3-extensions\&quot;,\&quot;druid-histogram\&quot;,\&quot;mysql-metadata-storage\&quot;] -Ddruid.historical.cache.useCache=false -Ddruid.historical.cache.populateCache=false -Ddruid.segmentCache.locations=&quot;[{\&quot;path\&quot;:\&quot;/tmp/druid/indexCache\&quot;,\&quot;maxSize\&quot;:10000000000}]&quot; -Ddruid.zk.service.host=localhost -Ddruid.processing.numThreads=1 -Ddruid.server.http.numThreads=50 -Ddruid.serverview.type=batch -Ddruid.emitter=logging" />
+    <option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx2G -XX:MaxJavaStackTraceDepth=9999 -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Dlog4j.configurationFile=$PROJECT_DIR$/core/src/main/resources/log4j2.debug.xml -Ddruid.host=localhost -Ddruid.service=historical -Ddruid.processing.buffer.sizeBytes=100000000 -Ddruid.extensions.hadoopDependenciesDir=$PROJECT_DIR$/distribution/target/hadoop-dependencies/ -Ddruid.extensions.directory=$PROJECT_DIR$/distribution/target/extensions/ -Ddruid.extensions.loadList=[\&quot;druid-s3-extensions\&quot;,\&quot;druid-histogram\&quot;,\&quot;mysql-metadata-storage\&quot;] -Ddruid.historical.cache.useCache=false -Ddruid.historical.cache.populateCache=false -Ddruid.segmentCache.locations=&quot;[{\&quot;path\&quot;:\&quot;/tmp/druid/indexCache\&quot;,\&quot;maxSize\&quot;:10000000000}]&quot; -Ddruid.zk.service.host=localhost -Ddruid.processing.numThreads=1 -Ddruid.server.http.numThreads=50 -Ddruid.serverview.type=http -Ddruid.emitter=logging" />
     <option name="PROGRAM_PARAMETERS" value="server historical" />
     <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
@@ -114,7 +114,7 @@ You can configure application definitions in XML for import into IntelliJ. Below
   <configuration default="false" name="Coordinator" type="Application" factoryName="Application">
     <extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
     <option name="MAIN_CLASS_NAME" value="org.apache.druid.cli.Main" />
-    <option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx256M -Xmx256M -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Ddruid.host=localhost -Ddruid.service=coordinator -Ddruid.extensions.directory=$PROJECT_DIR$/distribution/target/extensions/ -Ddruid.extensions.loadList=[\&quot;druid-s3-extensions\&quot;,\&quot;druid-histogram\&quot;,\&quot;mysql-metadata-storage\&quot;] -Ddruid.zk.service.host=localhost -Ddruid.metadata.storage.type=mysql -Ddruid.metadata.storage.connector.connectURI=&quot;jdbc:mysql://localhost:3306/druid&quot; -Ddruid.metadata.storage.connector.user=druid -Ddruid.metadata.storage.connector.password=diurd -Ddruid.serverview.type=batch -Ddruid.emitter=logging -Ddruid.coordinator.period=PT10S -Ddruid.coordinator.startDelay=PT5S" />
+    <option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx256M -Xmx256M -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Ddruid.host=localhost -Ddruid.service=coordinator -Ddruid.extensions.directory=$PROJECT_DIR$/distribution/target/extensions/ -Ddruid.extensions.loadList=[\&quot;druid-s3-extensions\&quot;,\&quot;druid-histogram\&quot;,\&quot;mysql-metadata-storage\&quot;] -Ddruid.zk.service.host=localhost -Ddruid.metadata.storage.type=mysql -Ddruid.metadata.storage.connector.connectURI=&quot;jdbc:mysql://localhost:3306/druid&quot; -Ddruid.metadata.storage.connector.user=druid -Ddruid.metadata.storage.connector.password=diurd -Ddruid.serverview.type=http -Ddruid.emitter=logging -Ddruid.coordinator.period=PT10S -Ddruid.coordinator.startDelay=PT5S" />
     <option name="PROGRAM_PARAMETERS" value="server coordinator" />
     <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index ad46f228194..38995c3a208 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -872,8 +872,8 @@ These Coordinator static configurations can be defined in the `coordinator/runti
 ##### Segment Management
 |Property|Possible Values|Description|Default|
 |--------|---------------|-----------|-------|
-|`druid.serverview.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of ZooKeeper.|batch|
-|`druid.coordinator.loadqueuepeon.type`|curator or http|Whether to use "http" or "curator" implementation to assign segment loads/drops to historical|curator|
+|`druid.serverview.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of ZooKeeper.|http|
+|`druid.coordinator.loadqueuepeon.type`|curator or http|Whether to use "http" or "curator" implementation to assign segment loads/drops to historical|http|
 |`druid.coordinator.segment.awaitInitializationOnStart`|true or false|Whether the Coordinator will wait for its view of segments to fully initialize before starting up. If set to 'true', the Coordinator's HTTP server will not start up, and the Coordinator will not announce itself as available, until the server view is initialized.|true|
 
 ###### Additional config when "http" loadqueuepeon is used
@@ -1098,7 +1098,7 @@ These Overlord static configurations can be defined in the `overlord/runtime.pro
 
 |Property|Description|Default|
 |--------|-----------|-------|
-|`druid.indexer.runner.type`|Choices "local" or "remote". Indicates whether tasks should be run locally or in a distributed environment. Experimental task runner "httpRemote" is also available which is same as "remote" but uses HTTP to interact with Middle Managers instead of ZooKeeper.|local|
+|`druid.indexer.runner.type`|Indicates whether tasks should be run locally using "local" or in a distributed environment using "remote". The recommended option is "httpRemote", which is similar to "remote" but uses HTTP to interact with Middle Managers instead of ZooKeeper.|httpRemote|
 |`druid.indexer.storage.type`|Choices are "local" or "metadata". Indicates whether incoming tasks should be stored locally (in heap) or in metadata storage. "local" is mainly for internal testing while "metadata" is recommended in production because storing incoming tasks in metadata storage allows for tasks to be resumed if the Overlord should fail.|local|
 |`druid.indexer.storage.recentlyFinishedThreshold`|Duration of time to store task results. Default is 24 hours. If you have hundreds of tasks running in a day, consider increasing this threshold.|PT24H|
 |`druid.indexer.tasklock.forceTimeChunkLock`|_**Setting this to false is still experimental**_<br/> If set, all tasks are enforced to use time chunk lock. If not set, each task automatically chooses a lock type to use. This configuration can be overwritten by setting `forceTimeChunkLock` in the [task context](../ingestion/tasks.md#context). See [Task Locking & Priority](../ingestion/tasks.md#context) for more details about locking in tasks.|true|
@@ -1930,7 +1930,7 @@ See [cache configuration](#cache-configuration) for how to configure cache setti
 #### Segment Discovery
 |Property|Possible Values|Description|Default|
 |--------|---------------|-----------|-------|
-|`druid.serverview.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of ZooKeeper.|batch|
+|`druid.serverview.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of ZooKeeper.|http|
 |`druid.broker.segment.watchedTiers`|List of strings|The Broker watches segment announcements from processes that serve segments to build a cache to relate each process to the segments it serves. This configuration allows the Broker to only consider segments being served from a list of tiers. By default, Broker considers all tiers. This can be used to partition your dataSources in specific Historical tiers and configure brokers in partitions so that they are only queryable for specific dataSources. This config is mutually exclusive from `druid.broker.segment.ignoredTiers` and at most one of these can be configured on a Broker.|none|
 |`druid.broker.segment.ignoredTiers`|List of strings|The Broker watches segment announcements from processes that serve segments to build a cache to relate each process to the segments it serves. This configuration allows the Broker to ignore the segments being served from a list of tiers. By default, Broker considers all tiers. This config is mutually exclusive from `druid.broker.segment.watchedTiers` and at most one of these can be configured on a Broker.|none|
 |`druid.broker.segment.watchedDataSources`|List of strings|Broker watches the segment announcements from processes serving segments to build cache of which process is serving which segments, this configuration allows to only consider segments being served from a whitelist of dataSources. By default, Broker would consider all datasources. This can be used to configure brokers in partitions so that they are only queryable for specific dataSources.|none|
diff --git a/examples/conf/druid/cluster/master/coordinator-overlord/runtime.properties b/examples/conf/druid/cluster/master/coordinator-overlord/runtime.properties
index 00071a83d7d..c053823f8ed 100644
--- a/examples/conf/druid/cluster/master/coordinator-overlord/runtime.properties
+++ b/examples/conf/druid/cluster/master/coordinator-overlord/runtime.properties
@@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
 
 druid.indexer.queue.startDelay=PT5S
 
-druid.indexer.runner.type=remote
 druid.indexer.storage.type=metadata
diff --git a/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties
index 00071a83d7d..c053823f8ed 100644
--- a/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties
+++ b/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties
@@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
 
 druid.indexer.queue.startDelay=PT5S
 
-druid.indexer.runner.type=remote
 druid.indexer.storage.type=metadata
diff --git a/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties
index 00071a83d7d..c053823f8ed 100644
--- a/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties
+++ b/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties
@@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
 
 druid.indexer.queue.startDelay=PT5S
 
-druid.indexer.runner.type=remote
 druid.indexer.storage.type=metadata
diff --git a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties
index 00071a83d7d..c053823f8ed 100644
--- a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties
+++ b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties
@@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
 
 druid.indexer.queue.startDelay=PT5S
 
-druid.indexer.runner.type=remote
 druid.indexer.storage.type=metadata
diff --git a/examples/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties
index 00071a83d7d..c053823f8ed 100644
--- a/examples/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties
+++ b/examples/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties
@@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
 
 druid.indexer.queue.startDelay=PT5S
 
-druid.indexer.runner.type=remote
 druid.indexer.storage.type=metadata
diff --git a/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties
index 00071a83d7d..c053823f8ed 100644
--- a/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties
+++ b/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties
@@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
 
 druid.indexer.queue.startDelay=PT5S
 
-druid.indexer.runner.type=remote
 druid.indexer.storage.type=metadata
diff --git a/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties
index 00071a83d7d..c053823f8ed 100644
--- a/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties
+++ b/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties
@@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
 
 druid.indexer.queue.startDelay=PT5S
 
-druid.indexer.runner.type=remote
 druid.indexer.storage.type=metadata
diff --git a/server/src/main/java/org/apache/druid/client/FilteredServerInventoryViewProvider.java b/server/src/main/java/org/apache/druid/client/FilteredServerInventoryViewProvider.java
index fdbb27e6382..26cd53ad199 100644
--- a/server/src/main/java/org/apache/druid/client/FilteredServerInventoryViewProvider.java
+++ b/server/src/main/java/org/apache/druid/client/FilteredServerInventoryViewProvider.java
@@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.inject.Provider;
 
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = FilteredBatchServerInventoryViewProvider.class)
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = FilteredHttpServerInventoryViewProvider.class)
 @JsonSubTypes(value = {
     @JsonSubTypes.Type(name = "batch", value = FilteredBatchServerInventoryViewProvider.class),
     @JsonSubTypes.Type(name = "http", value = FilteredHttpServerInventoryViewProvider.class)
diff --git a/server/src/main/java/org/apache/druid/client/ServerInventoryViewProvider.java b/server/src/main/java/org/apache/druid/client/ServerInventoryViewProvider.java
index bc87500f905..3bbf8557053 100644
--- a/server/src/main/java/org/apache/druid/client/ServerInventoryViewProvider.java
+++ b/server/src/main/java/org/apache/druid/client/ServerInventoryViewProvider.java
@@ -25,7 +25,7 @@ import com.google.inject.Provider;
 
 /**
  */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = BatchServerInventoryViewProvider.class)
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = HttpServerInventoryViewProvider.class)
 @JsonSubTypes(value = {
     @JsonSubTypes.Type(name = "batch", value = BatchServerInventoryViewProvider.class),
     @JsonSubTypes.Type(name = "http", value = HttpServerInventoryViewProvider.class),
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorConfig.java
index c02baf9cc64..cdd708ea9d3 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorConfig.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorConfig.java
@@ -110,7 +110,7 @@ public abstract class DruidCoordinatorConfig
   @Config("druid.coordinator.loadqueuepeon.type")
   public String getLoadQueuePeonType()
   {
-    return "curator";
+    return "http";
   }
 
   @Config("druid.coordinator.curator.loadqueuepeon.numCallbackThreads")
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorConfigTest.java
index 169ad39fa0d..cfbe2b7de97 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorConfigTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorConfigTest.java
@@ -49,6 +49,7 @@ public class DruidCoordinatorConfigTest
     Assert.assertEquals(Duration.millis(50), config.getLoadQueuePeonRepeatDelay());
     Assert.assertTrue(config.getCompactionSkipLockedIntervals());
     Assert.assertFalse(config.getCoordinatorKillIgnoreDurationToRetain());
+    Assert.assertEquals("http", config.getLoadQueuePeonType());
 
     //with non-defaults
     Properties props = new Properties();
diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java
index 3a45975bf9d..cb9b68788b4 100644
--- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java
+++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java
@@ -300,7 +300,7 @@ public class CliOverlord extends ServerRunnable
                 binder,
                 "druid.indexer.runner.type",
                 Key.get(TaskRunnerFactory.class),
-                Key.get(ForkingTaskRunnerFactory.class)
+                Key.get(HttpRemoteTaskRunnerFactory.class)
             );
             final MapBinder<String, TaskRunnerFactory> biddy = PolyBind.optionBinder(
                 binder,