mirror of https://github.com/apache/druid.git
Make http options the default configurations (#13092)
Druid currently uses Zookeeper dependent options as the default. This commit updates the following to use HTTP as the default instead. - task runner. `druid.indexer.runner.type=remote -> httpRemote` - load queue peon. `druid.coordinator.loadqueuepeon.type=curator -> http` - server inventory view. `druid.serverview.type=curator -> http`
This commit is contained in:
AmatyaAvadhanula
GitHub
parent
eff7edb603
commit
41e51b21c3
|
|
@ -93,7 +93,7 @@ You can configure application definitions in XML for import into IntelliJ. Below
|
|||
<configuration default="false" name="Historical" type="Application" factoryName="Application">
|
||||
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
|
||||
<option name="MAIN_CLASS_NAME" value="org.apache.druid.cli.Main" />
|
||||
<option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx2G -XX:MaxJavaStackTraceDepth=9999 -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Dlog4j.configurationFile=$PROJECT_DIR$/core/src/main/resources/log4j2.debug.xml -Ddruid.host=localhost -Ddruid.service=historical -Ddruid.processing.buffer.sizeBytes=100000000 -Ddruid.extensions.hadoopDependenciesDir=$PROJECT_DIR$/distribution/target/hadoop-dependencies/ -Ddruid.extensions.directory=$PROJECT_DIR$/distribution/target/extensions/ -Ddruid.extensions.loadList=[\"druid-s3-extensions\",\"druid-histogram\",\"mysql-metadata-storage\"] -Ddruid.historical.cache.useCache=false -Ddruid.historical.cache.populateCache=false -Ddruid.segmentCache.locations="[{\"path\":\"/tmp/druid/indexCache\",\"maxSize\":10000000000}]" -Ddruid.zk.service.host=localhost -Ddruid.processing.numThreads=1 -Ddruid.server.http.numThreads=50 -Ddruid.serverview.type=batch -Ddruid.emitter=logging" />
|
||||
<option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx2G -XX:MaxJavaStackTraceDepth=9999 -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Dlog4j.configurationFile=$PROJECT_DIR$/core/src/main/resources/log4j2.debug.xml -Ddruid.host=localhost -Ddruid.service=historical -Ddruid.processing.buffer.sizeBytes=100000000 -Ddruid.extensions.hadoopDependenciesDir=$PROJECT_DIR$/distribution/target/hadoop-dependencies/ -Ddruid.extensions.directory=$PROJECT_DIR$/distribution/target/extensions/ -Ddruid.extensions.loadList=[\"druid-s3-extensions\",\"druid-histogram\",\"mysql-metadata-storage\"] -Ddruid.historical.cache.useCache=false -Ddruid.historical.cache.populateCache=false -Ddruid.segmentCache.locations="[{\"path\":\"/tmp/druid/indexCache\",\"maxSize\":10000000000}]" -Ddruid.zk.service.host=localhost -Ddruid.processing.numThreads=1 -Ddruid.server.http.numThreads=50 -Ddruid.serverview.type=http -Ddruid.emitter=logging" />
|
||||
<option name="PROGRAM_PARAMETERS" value="server historical" />
|
||||
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
|
||||
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
|
||||
|
|
@ -114,7 +114,7 @@ You can configure application definitions in XML for import into IntelliJ. Below
|
|||
<configuration default="false" name="Coordinator" type="Application" factoryName="Application">
|
||||
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
|
||||
<option name="MAIN_CLASS_NAME" value="org.apache.druid.cli.Main" />
|
||||
<option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx256M -Xmx256M -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Ddruid.host=localhost -Ddruid.service=coordinator -Ddruid.extensions.directory=$PROJECT_DIR$/distribution/target/extensions/ -Ddruid.extensions.loadList=[\"druid-s3-extensions\",\"druid-histogram\",\"mysql-metadata-storage\"] -Ddruid.zk.service.host=localhost -Ddruid.metadata.storage.type=mysql -Ddruid.metadata.storage.connector.connectURI="jdbc:mysql://localhost:3306/druid" -Ddruid.metadata.storage.connector.user=druid -Ddruid.metadata.storage.connector.password=diurd -Ddruid.serverview.type=batch -Ddruid.emitter=logging -Ddruid.coordinator.period=PT10S -Ddruid.coordinator.startDelay=PT5S" />
|
||||
<option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx256M -Xmx256M -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Ddruid.host=localhost -Ddruid.service=coordinator -Ddruid.extensions.directory=$PROJECT_DIR$/distribution/target/extensions/ -Ddruid.extensions.loadList=[\"druid-s3-extensions\",\"druid-histogram\",\"mysql-metadata-storage\"] -Ddruid.zk.service.host=localhost -Ddruid.metadata.storage.type=mysql -Ddruid.metadata.storage.connector.connectURI="jdbc:mysql://localhost:3306/druid" -Ddruid.metadata.storage.connector.user=druid -Ddruid.metadata.storage.connector.password=diurd -Ddruid.serverview.type=http -Ddruid.emitter=logging -Ddruid.coordinator.period=PT10S -Ddruid.coordinator.startDelay=PT5S" />
|
||||
<option name="PROGRAM_PARAMETERS" value="server coordinator" />
|
||||
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
|
||||
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
|
||||
|
|
|
|||
|
|
@ -872,8 +872,8 @@ These Coordinator static configurations can be defined in the `coordinator/runti
|
|||
##### Segment Management
|
||||
|Property|Possible Values|Description|Default|
|
||||
|--------|---------------|-----------|-------|
|
||||
|`druid.serverview.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of ZooKeeper.|batch|
|
||||
|`druid.coordinator.loadqueuepeon.type`|curator or http|Whether to use "http" or "curator" implementation to assign segment loads/drops to historical|curator|
|
||||
|`druid.serverview.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of ZooKeeper.|http|
|
||||
|`druid.coordinator.loadqueuepeon.type`|curator or http|Whether to use "http" or "curator" implementation to assign segment loads/drops to historical|http|
|
||||
|`druid.coordinator.segment.awaitInitializationOnStart`|true or false|Whether the Coordinator will wait for its view of segments to fully initialize before starting up. If set to 'true', the Coordinator's HTTP server will not start up, and the Coordinator will not announce itself as available, until the server view is initialized.|true|
|
||||
|
||||
###### Additional config when "http" loadqueuepeon is used
|
||||
|
|
@ -1098,7 +1098,7 @@ These Overlord static configurations can be defined in the `overlord/runtime.pro
|
|||
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.indexer.runner.type`|Choices "local" or "remote". Indicates whether tasks should be run locally or in a distributed environment. Experimental task runner "httpRemote" is also available which is same as "remote" but uses HTTP to interact with Middle Managers instead of ZooKeeper.|local|
|
||||
|`druid.indexer.runner.type`|Indicates whether tasks should be run locally using "local" or in a distributed environment using "remote". The recommended option is "httpRemote", which is similar to "remote" but uses HTTP to interact with Middle Managers instead of ZooKeeper.|httpRemote|
|
||||
|`druid.indexer.storage.type`|Choices are "local" or "metadata". Indicates whether incoming tasks should be stored locally (in heap) or in metadata storage. "local" is mainly for internal testing while "metadata" is recommended in production because storing incoming tasks in metadata storage allows for tasks to be resumed if the Overlord should fail.|local|
|
||||
|`druid.indexer.storage.recentlyFinishedThreshold`|Duration of time to store task results. Default is 24 hours. If you have hundreds of tasks running in a day, consider increasing this threshold.|PT24H|
|
||||
|`druid.indexer.tasklock.forceTimeChunkLock`|_**Setting this to false is still experimental**_<br/> If set, all tasks are enforced to use time chunk lock. If not set, each task automatically chooses a lock type to use. This configuration can be overwritten by setting `forceTimeChunkLock` in the [task context](../ingestion/tasks.md#context). See [Task Locking & Priority](../ingestion/tasks.md#context) for more details about locking in tasks.|true|
|
||||
|
|
@ -1930,7 +1930,7 @@ See [cache configuration](#cache-configuration) for how to configure cache setti
|
|||
#### Segment Discovery
|
||||
|Property|Possible Values|Description|Default|
|
||||
|--------|---------------|-----------|-------|
|
||||
|`druid.serverview.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of ZooKeeper.|batch|
|
||||
|`druid.serverview.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of ZooKeeper.|http|
|
||||
|`druid.broker.segment.watchedTiers`|List of strings|The Broker watches segment announcements from processes that serve segments to build a cache to relate each process to the segments it serves. This configuration allows the Broker to only consider segments being served from a list of tiers. By default, Broker considers all tiers. This can be used to partition your dataSources in specific Historical tiers and configure brokers in partitions so that they are only queryable for specific dataSources. This config is mutually exclusive from `druid.broker.segment.ignoredTiers` and at most one of these can be configured on a Broker.|none|
|
||||
|`druid.broker.segment.ignoredTiers`|List of strings|The Broker watches segment announcements from processes that serve segments to build a cache to relate each process to the segments it serves. This configuration allows the Broker to ignore the segments being served from a list of tiers. By default, Broker considers all tiers. This config is mutually exclusive from `druid.broker.segment.watchedTiers` and at most one of these can be configured on a Broker.|none|
|
||||
|`druid.broker.segment.watchedDataSources`|List of strings|Broker watches the segment announcements from processes serving segments to build cache of which process is serving which segments, this configuration allows to only consider segments being served from a whitelist of dataSources. By default, Broker would consider all datasources. This can be used to configure brokers in partitions so that they are only queryable for specific dataSources.|none|
|
||||
|
|
|
|||
|
|
@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
|
|||
|
||||
druid.indexer.queue.startDelay=PT5S
|
||||
|
||||
druid.indexer.runner.type=remote
|
||||
druid.indexer.storage.type=metadata
|
||||
|
|
|
|||
|
|
@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
|
|||
|
||||
druid.indexer.queue.startDelay=PT5S
|
||||
|
||||
druid.indexer.runner.type=remote
|
||||
druid.indexer.storage.type=metadata
|
||||
|
|
|
|||
|
|
@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
|
|||
|
||||
druid.indexer.queue.startDelay=PT5S
|
||||
|
||||
druid.indexer.runner.type=remote
|
||||
druid.indexer.storage.type=metadata
|
||||
|
|
|
|||
|
|
@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
|
|||
|
||||
druid.indexer.queue.startDelay=PT5S
|
||||
|
||||
druid.indexer.runner.type=remote
|
||||
druid.indexer.storage.type=metadata
|
||||
|
|
|
|||
|
|
@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
|
|||
|
||||
druid.indexer.queue.startDelay=PT5S
|
||||
|
||||
druid.indexer.runner.type=remote
|
||||
druid.indexer.storage.type=metadata
|
||||
|
|
|
|||
|
|
@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
|
|||
|
||||
druid.indexer.queue.startDelay=PT5S
|
||||
|
||||
druid.indexer.runner.type=remote
|
||||
druid.indexer.storage.type=metadata
|
||||
|
|
|
|||
|
|
@ -30,5 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord
|
|||
|
||||
druid.indexer.queue.startDelay=PT5S
|
||||
|
||||
druid.indexer.runner.type=remote
|
||||
druid.indexer.storage.type=metadata
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
|
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo;
|
||||
import com.google.inject.Provider;
|
||||
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = FilteredBatchServerInventoryViewProvider.class)
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = FilteredHttpServerInventoryViewProvider.class)
|
||||
@JsonSubTypes(value = {
|
||||
@JsonSubTypes.Type(name = "batch", value = FilteredBatchServerInventoryViewProvider.class),
|
||||
@JsonSubTypes.Type(name = "http", value = FilteredHttpServerInventoryViewProvider.class)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import com.google.inject.Provider;
|
|||
|
||||
/**
|
||||
*/
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = BatchServerInventoryViewProvider.class)
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = HttpServerInventoryViewProvider.class)
|
||||
@JsonSubTypes(value = {
|
||||
@JsonSubTypes.Type(name = "batch", value = BatchServerInventoryViewProvider.class),
|
||||
@JsonSubTypes.Type(name = "http", value = HttpServerInventoryViewProvider.class),
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ public abstract class DruidCoordinatorConfig
|
|||
@Config("druid.coordinator.loadqueuepeon.type")
|
||||
public String getLoadQueuePeonType()
|
||||
{
|
||||
return "curator";
|
||||
return "http";
|
||||
}
|
||||
|
||||
@Config("druid.coordinator.curator.loadqueuepeon.numCallbackThreads")
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ public class DruidCoordinatorConfigTest
|
|||
Assert.assertEquals(Duration.millis(50), config.getLoadQueuePeonRepeatDelay());
|
||||
Assert.assertTrue(config.getCompactionSkipLockedIntervals());
|
||||
Assert.assertFalse(config.getCoordinatorKillIgnoreDurationToRetain());
|
||||
Assert.assertEquals("http", config.getLoadQueuePeonType());
|
||||
|
||||
//with non-defaults
|
||||
Properties props = new Properties();
|
||||
|
|
|
|||
|
|
@ -300,7 +300,7 @@ public class CliOverlord extends ServerRunnable
|
|||
binder,
|
||||
"druid.indexer.runner.type",
|
||||
Key.get(TaskRunnerFactory.class),
|
||||
Key.get(ForkingTaskRunnerFactory.class)
|
||||
Key.get(HttpRemoteTaskRunnerFactory.class)
|
||||
);
|
||||
final MapBinder<String, TaskRunnerFactory> biddy = PolyBind.optionBinder(
|
||||
binder,
|
||||
|
|
|
|||
Loading…
Reference in New Issue