diff --git a/distribution/src/assembly/assembly.xml b/distribution/src/assembly/assembly.xml index 856f5931608..854f41cac2b 100644 --- a/distribution/src/assembly/assembly.xml +++ b/distribution/src/assembly/assembly.xml @@ -41,7 +41,17 @@ hadoop-dependencies - + + ../examples/conf + + * + */* + */*/* + */*/*/* + */*/*/*/* + + conf + ../examples/quickstart/ @@ -56,83 +66,6 @@ quickstart/tutorial - - ../examples/quickstart/tutorial/conf - - * - - quickstart/tutorial/conf - - - ../examples/quickstart/tutorial/conf/druid - - * - - quickstart/tutorial/conf/druid - - - ../examples/quickstart/tutorial/conf/druid/_common - - * - - quickstart/tutorial/conf/druid/_common/ - - - ../examples/quickstart/tutorial/conf/druid/broker - - * - - quickstart/tutorial/conf/druid/broker - - - ../examples/quickstart/tutorial/conf/druid/coordinator - - * - - quickstart/tutorial/conf/druid/coordinator - - - ../examples/quickstart/tutorial/conf/druid/historical - - * - - quickstart/tutorial/conf/druid/historical - - - ../examples/quickstart/tutorial/conf/druid/overlord - - * - - quickstart/tutorial/conf/druid/overlord - - - ../examples/quickstart/tutorial/conf/druid/middleManager - - * - - quickstart/tutorial/conf/druid/middleManager - - - ../examples/quickstart/tutorial/conf/druid/router - - * - - quickstart/tutorial/conf/druid/router - - - ../examples/quickstart/tutorial/conf/tranquility - - * - - quickstart/tutorial/conf/tranquility - - - ../examples/quickstart/tutorial/conf/zk - - * - - quickstart/tutorial/conf/zk - ../examples/quickstart/tutorial/hadoop @@ -147,8 +80,6 @@ quickstart/tutorial/hadoop/docker - - ../examples/quickstart/protobuf @@ -156,70 +87,6 @@ quickstart/protobuf - - - ../examples/conf - - * - - conf - - - ../examples/conf/druid/_common - - * - - conf/druid/_common - - - ../examples/conf/druid/broker - - * - - conf/druid/broker - - - ../examples/conf/druid/coordinator - - * - - conf/druid/coordinator - - - ../examples/conf/druid/historical - - * - - conf/druid/historical - - - ../examples/conf/druid/overlord - - * - - conf/druid/overlord - - - ../examples/conf/druid/middleManager - - * - - conf/druid/middleManager - - - ../examples/conf/druid/router - - * - - conf/druid/router - - - ../examples/conf/tranquility - - * - - conf/tranquility - ../examples/bin @@ -228,7 +95,6 @@ 744 bin - ../ diff --git a/docs/content/operations/single-server.md b/docs/content/operations/single-server.md new file mode 100644 index 00000000000..62ae8f95361 --- /dev/null +++ b/docs/content/operations/single-server.md @@ -0,0 +1,69 @@ +--- +layout: doc_page +title: "Single Server Deployments" +--- + + + +# Single Server Deployments + +Druid includes a set of reference configurations and launch scripts for single-machine deployments: + +- `micro-quickstart` +- `small` +- `medium` +- `large` +- `xlarge` + +The `micro-quickstart` is sized for small machines like laptops and is intended for quick evaluation use-cases. + +The other configurations are intended for general use single-machine deployments. They are sized for hardware roughly based on Amazon's i3 series of EC2 instances. + +The startup scripts for these example configurations run a single ZK instance along with the Druid services. You can choose to deploy ZK separately as well. + +The example configurations run the Druid Coordinator and Overlord together in a single process using the optional configuration `druid.coordinator.asOverlord.enabled=true`, described in the [Coordinator configuration documentation](../configuration/index.html#coordinator-operation). + +## Single Server Reference Configurations + +Micro-Quickstart: 4 CPU, 16GB RAM +------------ +Launch command: `bin/start-micro-quickstart` +Configuration directory: `conf/druid/single-server/micro-quickstart` + +Small: 8 CPU, 64GB RAM (~i3.2xlarge) +------------ +Launch command: `bin/start-small` +Configuration directory: `conf/druid/single-server/small` + +Medium: 16 CPU, 128GB RAM (~i3.4xlarge) +------------ +Launch command: `bin/start-medium` +Configuration directory: `conf/druid/single-server/medium` + +Large: 32 CPU, 256GB RAM (~i3.8xlarge) +------------ +Launch command: `bin/start-large` +Configuration directory: `conf/druid/single-server/large` + +X-Large: 64 CPU, 512GB RAM (~i3.16xlarge) +------------ +Launch command: `bin/start-xlarge` +Configuration directory: `conf/druid/single-server/xlarge` + diff --git a/docs/content/tutorials/index.md b/docs/content/tutorials/index.md index 9f791655a96..8d99fb70d78 100644 --- a/docs/content/tutorials/index.md +++ b/docs/content/tutorials/index.md @@ -22,7 +22,7 @@ title: "Apache Druid (incubating) Quickstart" ~ under the License. --> -# Druid Quickstart +# Apache Druid (incubating) Quickstart In this quickstart, we will download Druid and set it up on a single machine. The cluster will be ready to load data after completing this initial setup. @@ -32,13 +32,12 @@ Before beginning the quickstart, it is helpful to read the [general Druid overvi ## Prerequisites +### Software + You will need: - - * Java 8 + * Java 8 (8u92+) * Linux, Mac OS X, or other Unix-like OS (Windows is not supported) - * 8G of RAM - * 2 vCPUs - + On Mac OS X, you can use [Oracle's JDK 8](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) to install Java. @@ -47,6 +46,15 @@ On Linux, your OS package manager should be able to help for Java. If your Ubunt based OS does not have a recent enough version of Java, WebUpd8 offers [packages for those OSes](http://www.webupd8.org/2012/09/install-oracle-java-8-in-ubuntu-via-ppa.html). + +### Hardware + +Druid includes several example [single-server configurations](../operations/single-server.html), along with scripts to start the Druid processes using these configurations. + +If you're running on a small machine such as a laptop for a quick evaluation, the `micro-quickstart` configuration is a good choice, sized for a 4CPU/16GB RAM environment. + +If you plan to use the single-machine deployment for further evaluation beyond the tutorials, we recommend a larger configuration than `micro-quickstart`. + ## Getting started [Download](https://www.apache.org/dyn/closer.cgi?path=/incubator/druid/#{DRUIDVERSION}/apache-druid-#{DRUIDVERSION}-bin.tar.gz) @@ -86,32 +94,33 @@ The startup scripts for the tutorial will expect the contents of the Zookeeper t ## Start up Druid services +The following commands will assume that you are using the `micro-quickstart` single-machine configuration. If you are using a different configuration, the `bin` directory has equivalent scripts for each configuration, such as `bin/start-single-server-small`. + From the apache-druid-#{DRUIDVERSION} package root, run the following command: ```bash -bin/supervise -c quickstart/tutorial/conf/tutorial-cluster.conf +./bin/start-micro-quickstart ``` This will bring up instances of Zookeeper and the Druid services, all running on the local machine, e.g.: ```bash -bin/supervise -c quickstart/tutorial/conf/tutorial-cluster.conf -[Wed Feb 27 12:46:13 2019] Running command[zk], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/zk.log]: bin/run-zk quickstart/tutorial/conf -[Wed Feb 27 12:46:13 2019] Running command[coordinator], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/coordinator.log]: bin/run-druid coordinator quickstart/tutorial/conf -[Wed Feb 27 12:46:13 2019] Running command[broker], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/broker.log]: bin/run-druid broker quickstart/tutorial/conf -[Wed Feb 27 12:46:13 2019] Running command[router], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/router.log]: bin/run-druid router quickstart/tutorial/conf -[Wed Feb 27 12:46:13 2019] Running command[historical], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/historical.log]: bin/run-druid historical quickstart/tutorial/conf -[Wed Feb 27 12:46:13 2019] Running command[overlord], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/overlord.log]: bin/run-druid overlord quickstart/tutorial/conf -[Wed Feb 27 12:46:13 2019] Running command[middleManager], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/middleManager.log]: bin/run-druid middleManager quickstart/tutorial/conf +$ ./bin/start-micro-quickstart +[Fri May 3 11:40:50 2019] Running command[zk], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/zk.log]: bin/run-zk conf +[Fri May 3 11:40:50 2019] Running command[coordinator-overlord], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/coordinator-overlord.log]: bin/run-druid coordinator-overlord conf/druid/single-server/micro-quickstart +[Fri May 3 11:40:50 2019] Running command[broker], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/broker.log]: bin/run-druid broker conf/druid/single-server/micro-quickstart +[Fri May 3 11:40:50 2019] Running command[router], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/router.log]: bin/run-druid router conf/druid/single-server/micro-quickstart +[Fri May 3 11:40:50 2019] Running command[historical], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/historical.log]: bin/run-druid historical conf/druid/single-server/micro-quickstart +[Fri May 3 11:40:50 2019] Running command[middleManager], logging to[/apache-druid-#{DRUIDVERSION}/var/sv/middleManager.log]: bin/run-druid middleManager conf/druid/single-server/micro-quickstart ``` All persistent state such as the cluster metadata store and segments for the services will be kept in the `var` directory under the apache-druid-#{DRUIDVERSION} package root. Logs for the services are located at `var/sv`. -Later on, if you'd like to stop the services, CTRL-C to exit the `bin/supervise` script, which will terminate the Druid processes. +Later on, if you'd like to stop the services, CTRL-C to exit the `bin/start-micro-quickstart` script, which will terminate the Druid processes. ### Resetting cluster state -If you want a clean start after stopping the services, delete the `var` directory and run the `bin/supervise` script again. +If you want a clean start after stopping the services, delete the `var` directory and run the `bin/start-micro-quickstart` script again. Once every service has started, you are now ready to load data. diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 703b2d58c00..82695f60f87 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -34,7 +34,7 @@ else CONFDIR="$2" fi -CONFDIR="$(cd "$CONFDIR" && pwd)/druid" +CONFDIR="$(cd "$CONFDIR" && pwd)" WHEREAMI="$(cd "$WHEREAMI" && pwd)" cd "$WHEREAMI/.." diff --git a/examples/conf/druid/overlord/runtime.properties b/examples/bin/start-cluster-data-server old mode 100644 new mode 100755 similarity index 82% rename from examples/conf/druid/overlord/runtime.properties rename to examples/bin/start-cluster-data-server index 71222efb77e..83af3d82a30 --- a/examples/conf/druid/overlord/runtime.properties +++ b/examples/bin/start-cluster-data-server @@ -1,4 +1,5 @@ -# +#!/bin/bash -eu + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +16,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -druid.service=druid/overlord -druid.plaintextPort=8090 +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" -druid.indexer.queue.startDelay=PT30S - -druid.indexer.runner.type=remote -druid.indexer.storage.type=metadata +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/cluster/data.conf" diff --git a/examples/bin/start-cluster-master-no-zk-server b/examples/bin/start-cluster-master-no-zk-server new file mode 100755 index 00000000000..3bc69f65801 --- /dev/null +++ b/examples/bin/start-cluster-master-no-zk-server @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/cluster/master-no-zk.conf" diff --git a/examples/bin/start-cluster-master-with-zk-server b/examples/bin/start-cluster-master-with-zk-server new file mode 100755 index 00000000000..337b1a396cf --- /dev/null +++ b/examples/bin/start-cluster-master-with-zk-server @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/cluster/master-with-zk.conf" diff --git a/examples/conf/druid/coordinator/runtime.properties b/examples/bin/start-cluster-query-server old mode 100644 new mode 100755 similarity index 82% rename from examples/conf/druid/coordinator/runtime.properties rename to examples/bin/start-cluster-query-server index 25e07643713..09c674b9d08 --- a/examples/conf/druid/coordinator/runtime.properties +++ b/examples/bin/start-cluster-query-server @@ -1,4 +1,5 @@ -# +#!/bin/bash -eu + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,10 +16,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -druid.service=druid/coordinator -druid.plaintextPort=8081 +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" -druid.coordinator.startDelay=PT30S -druid.coordinator.period=PT30S +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/cluster/query.conf" diff --git a/examples/bin/start-micro-quickstart b/examples/bin/start-micro-quickstart new file mode 100755 index 00000000000..bdad0d13659 --- /dev/null +++ b/examples/bin/start-micro-quickstart @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/single-server/micro-quickstart.conf" diff --git a/examples/bin/start-single-server-large b/examples/bin/start-single-server-large new file mode 100755 index 00000000000..1384bd300d0 --- /dev/null +++ b/examples/bin/start-single-server-large @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/single-server/large.conf" diff --git a/examples/bin/start-single-server-medium b/examples/bin/start-single-server-medium new file mode 100755 index 00000000000..da4626626f1 --- /dev/null +++ b/examples/bin/start-single-server-medium @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/single-server/medium.conf" diff --git a/examples/bin/start-single-server-small b/examples/bin/start-single-server-small new file mode 100755 index 00000000000..c308076611c --- /dev/null +++ b/examples/bin/start-single-server-small @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/single-server/small.conf" diff --git a/examples/bin/start-single-server-xlarge b/examples/bin/start-single-server-xlarge new file mode 100755 index 00000000000..c47aaa7591d --- /dev/null +++ b/examples/bin/start-single-server-xlarge @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +exec "$WHEREAMI/supervise" -c "$WHEREAMI/../conf/supervise/single-server/xlarge.conf" diff --git a/examples/conf/druid/_common/log4j2.xml b/examples/conf/druid/_common/log4j2.xml deleted file mode 100644 index 0e0f02f2ca6..00000000000 --- a/examples/conf/druid/_common/log4j2.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/examples/quickstart/tutorial/conf/druid/_common/common.runtime.properties b/examples/conf/druid/cluster/_common/common.runtime.properties similarity index 99% rename from examples/quickstart/tutorial/conf/druid/_common/common.runtime.properties rename to examples/conf/druid/cluster/_common/common.runtime.properties index 9d0f087d58a..6dd696f1f02 100644 --- a/examples/quickstart/tutorial/conf/druid/_common/common.runtime.properties +++ b/examples/conf/druid/cluster/_common/common.runtime.properties @@ -24,7 +24,7 @@ # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. # More info: http://druid.io/docs/latest/operations/including-extensions.html -druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service"] +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"] # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory # and uncomment the line below to point to your directory. diff --git a/examples/quickstart/tutorial/conf/druid/_common/log4j2.xml b/examples/conf/druid/cluster/_common/log4j2.xml similarity index 100% rename from examples/quickstart/tutorial/conf/druid/_common/log4j2.xml rename to examples/conf/druid/cluster/_common/log4j2.xml diff --git a/examples/conf/druid/historical/jvm.config b/examples/conf/druid/cluster/data/historical/jvm.config similarity index 86% rename from examples/conf/druid/historical/jvm.config rename to examples/conf/druid/cluster/data/historical/jvm.config index 94685fef5b3..3141abd754a 100644 --- a/examples/conf/druid/historical/jvm.config +++ b/examples/conf/druid/cluster/data/historical/jvm.config @@ -1,7 +1,7 @@ -server -Xms8g -Xmx8g --XX:MaxDirectMemorySize=4096m +-XX:MaxDirectMemorySize=14g -XX:+ExitOnOutOfMemoryError -Duser.timezone=UTC -Dfile.encoding=UTF-8 diff --git a/examples/quickstart/tutorial/conf/druid/historical/main.config b/examples/conf/druid/cluster/data/historical/main.config similarity index 100% rename from examples/quickstart/tutorial/conf/druid/historical/main.config rename to examples/conf/druid/cluster/data/historical/main.config diff --git a/examples/conf/druid/historical/runtime.properties b/examples/conf/druid/cluster/data/historical/runtime.properties similarity index 82% rename from examples/conf/druid/historical/runtime.properties rename to examples/conf/druid/cluster/data/historical/runtime.properties index e61e51c932d..5ee3a1c211e 100644 --- a/examples/conf/druid/historical/runtime.properties +++ b/examples/conf/druid/cluster/data/historical/runtime.properties @@ -21,15 +21,17 @@ druid.service=druid/historical druid.plaintextPort=8083 # HTTP server threads -druid.server.http.numThreads=25 +druid.server.http.numThreads=60 # Processing threads and buffers -druid.processing.buffer.sizeBytes=536870912 -druid.processing.numThreads=7 +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=4 +druid.processing.numThreads=16 +druid.processing.tmpDir=var/druid/processing # Segment storage -druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":130000000000}] -druid.server.maxSize=130000000000 +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}] +druid.server.maxSize=300000000000 # Query cache druid.historical.cache.useCache=true diff --git a/examples/quickstart/tutorial/conf/druid/overlord/jvm.config b/examples/conf/druid/cluster/data/middleManager/jvm.config similarity index 100% rename from examples/quickstart/tutorial/conf/druid/overlord/jvm.config rename to examples/conf/druid/cluster/data/middleManager/jvm.config diff --git a/examples/quickstart/tutorial/conf/druid/middleManager/main.config b/examples/conf/druid/cluster/data/middleManager/main.config similarity index 100% rename from examples/quickstart/tutorial/conf/druid/middleManager/main.config rename to examples/conf/druid/cluster/data/middleManager/main.config diff --git a/examples/conf/druid/middleManager/runtime.properties b/examples/conf/druid/cluster/data/middleManager/runtime.properties similarity index 71% rename from examples/conf/druid/middleManager/runtime.properties rename to examples/conf/druid/cluster/data/middleManager/runtime.properties index 8f9e8cb621f..8806fd1a27e 100644 --- a/examples/conf/druid/middleManager/runtime.properties +++ b/examples/conf/druid/cluster/data/middleManager/runtime.properties @@ -21,18 +21,19 @@ druid.service=druid/middleManager druid.plaintextPort=8091 # Number of tasks per middleManager -druid.worker.capacity=3 +druid.worker.capacity=4 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads -druid.server.http.numThreads=25 +druid.server.http.numThreads=60 # Processing threads and buffers on Peons -druid.indexer.fork.property.druid.processing.buffer.sizeBytes=536870912 -druid.indexer.fork.property.druid.processing.numThreads=2 +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=500000000 +druid.indexer.fork.property.druid.processing.numThreads=1 # Hadoop indexing druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/coordinator/jvm.config b/examples/conf/druid/cluster/master/coordinator/jvm.config similarity index 93% rename from examples/conf/druid/coordinator/jvm.config rename to examples/conf/druid/cluster/master/coordinator/jvm.config index a57248dcfde..084add76057 100644 --- a/examples/conf/druid/coordinator/jvm.config +++ b/examples/conf/druid/cluster/master/coordinator/jvm.config @@ -1,6 +1,6 @@ -server --Xms3g --Xmx3g +-Xms1g +-Xmx1g -XX:+ExitOnOutOfMemoryError -Duser.timezone=UTC -Dfile.encoding=UTF-8 diff --git a/examples/quickstart/tutorial/conf/druid/coordinator/main.config b/examples/conf/druid/cluster/master/coordinator/main.config similarity index 100% rename from examples/quickstart/tutorial/conf/druid/coordinator/main.config rename to examples/conf/druid/cluster/master/coordinator/main.config diff --git a/examples/quickstart/tutorial/conf/druid/coordinator/runtime.properties b/examples/conf/druid/cluster/master/coordinator/runtime.properties similarity index 100% rename from examples/quickstart/tutorial/conf/druid/coordinator/runtime.properties rename to examples/conf/druid/cluster/master/coordinator/runtime.properties diff --git a/examples/conf/druid/overlord/jvm.config b/examples/conf/druid/cluster/master/overlord/jvm.config similarity index 92% rename from examples/conf/druid/overlord/jvm.config rename to examples/conf/druid/cluster/master/overlord/jvm.config index 59e2eaa3c55..2bb6641a778 100644 --- a/examples/conf/druid/overlord/jvm.config +++ b/examples/conf/druid/cluster/master/overlord/jvm.config @@ -1,6 +1,6 @@ -server --Xms3g --Xmx3g +-Xms1g +-Xmx1g -XX:+ExitOnOutOfMemoryError -Duser.timezone=UTC -Dfile.encoding=UTF-8 diff --git a/examples/quickstart/tutorial/conf/druid/overlord/main.config b/examples/conf/druid/cluster/master/overlord/main.config similarity index 100% rename from examples/quickstart/tutorial/conf/druid/overlord/main.config rename to examples/conf/druid/cluster/master/overlord/main.config diff --git a/examples/quickstart/tutorial/conf/druid/overlord/runtime.properties b/examples/conf/druid/cluster/master/overlord/runtime.properties similarity index 100% rename from examples/quickstart/tutorial/conf/druid/overlord/runtime.properties rename to examples/conf/druid/cluster/master/overlord/runtime.properties diff --git a/examples/conf/druid/broker/jvm.config b/examples/conf/druid/cluster/query/broker/jvm.config similarity index 86% rename from examples/conf/druid/broker/jvm.config rename to examples/conf/druid/cluster/query/broker/jvm.config index cf67f93796d..a66f7513f98 100644 --- a/examples/conf/druid/broker/jvm.config +++ b/examples/conf/druid/cluster/query/broker/jvm.config @@ -1,7 +1,7 @@ -server -Xms24g -Xmx24g --XX:MaxDirectMemorySize=4096m +-XX:MaxDirectMemorySize=12g -XX:+ExitOnOutOfMemoryError -Duser.timezone=UTC -Dfile.encoding=UTF-8 diff --git a/examples/quickstart/tutorial/conf/druid/broker/main.config b/examples/conf/druid/cluster/query/broker/main.config similarity index 100% rename from examples/quickstart/tutorial/conf/druid/broker/main.config rename to examples/conf/druid/cluster/query/broker/main.config diff --git a/examples/conf/druid/cluster/query/broker/runtime.properties b/examples/conf/druid/cluster/query/broker/runtime.properties new file mode 100644 index 00000000000..6d4b3699fd8 --- /dev/null +++ b/examples/conf/druid/cluster/query/broker/runtime.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=5000000 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=16 +druid.processing.numThreads=1 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/examples/conf/druid/cluster/query/router/jvm.config b/examples/conf/druid/cluster/query/router/jvm.config new file mode 100644 index 00000000000..a844c40b1ea --- /dev/null +++ b/examples/conf/druid/cluster/query/router/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms1g +-Xmx1g +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/quickstart/tutorial/conf/druid/router/main.config b/examples/conf/druid/cluster/query/router/main.config similarity index 100% rename from examples/quickstart/tutorial/conf/druid/router/main.config rename to examples/conf/druid/cluster/query/router/main.config diff --git a/examples/conf/druid/router/runtime.properties b/examples/conf/druid/cluster/query/router/runtime.properties similarity index 100% rename from examples/conf/druid/router/runtime.properties rename to examples/conf/druid/cluster/query/router/runtime.properties diff --git a/examples/conf/druid/_common/common.runtime.properties b/examples/conf/druid/single-server/large/_common/common.runtime.properties similarity index 73% rename from examples/conf/druid/_common/common.runtime.properties rename to examples/conf/druid/single-server/large/_common/common.runtime.properties index 9db606026d8..6dd696f1f02 100644 --- a/examples/conf/druid/_common/common.runtime.properties +++ b/examples/conf/druid/single-server/large/_common/common.runtime.properties @@ -17,18 +17,25 @@ # under the License. # -# -# Extensions -# +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead -# This is not the full list of Druid extensions, but common ones that people often use. You may need to change this list -# based on your particular setup. -druid.extensions.loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global"] +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: http://druid.io/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"] # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory # and uncomment the line below to point to your directory. #druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + +# +# Hostname +# +druid.host=localhost + # # Logging # @@ -40,7 +47,7 @@ druid.startup.logging.logProperties=true # Zookeeper # -druid.zk.service.host=zk.host.ip +druid.zk.service.host=localhost druid.zk.paths.base=/druid # @@ -49,8 +56,8 @@ druid.zk.paths.base=/druid # For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): druid.metadata.storage.type=derby -druid.metadata.storage.connector.connectURI=jdbc:derby://metadata.store.ip:1527/var/druid/metadata.db;create=true -druid.metadata.storage.connector.host=metadata.store.ip +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost druid.metadata.storage.connector.port=1527 # For MySQL (make sure to include the MySQL JDBC driver on the classpath): @@ -59,7 +66,7 @@ druid.metadata.storage.connector.port=1527 #druid.metadata.storage.connector.user=... #druid.metadata.storage.connector.password=... -# For PostgreSQL (make sure to additionally include the Postgres extension): +# For PostgreSQL: #druid.metadata.storage.type=postgresql #druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid #druid.metadata.storage.connector.user=... @@ -73,7 +80,7 @@ druid.metadata.storage.connector.port=1527 druid.storage.type=local druid.storage.storageDirectory=var/druid/segments -# For HDFS (make sure to include the HDFS extension and that your Hadoop config files in the cp): +# For HDFS: #druid.storage.type=hdfs #druid.storage.storageDirectory=/druid/segments @@ -92,7 +99,7 @@ druid.storage.storageDirectory=var/druid/segments druid.indexer.logs.type=file druid.indexer.logs.directory=var/druid/indexing-logs -# For HDFS (make sure to include the HDFS extension and that your Hadoop config files in the cp): +# For HDFS: #druid.indexer.logs.type=hdfs #druid.indexer.logs.directory=/druid/indexing-logs @@ -113,7 +120,7 @@ druid.selectors.coordinator.serviceName=druid/coordinator # druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] -druid.emitter=logging +druid.emitter=noop druid.emitter.logging.logLevel=info # Storage type of double columns @@ -121,7 +128,18 @@ druid.emitter.logging.logLevel=info druid.indexing.doubleStorage=double +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password"] + + # # SQL # druid.sql.enable=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false \ No newline at end of file diff --git a/examples/conf/druid/single-server/large/_common/log4j2.xml b/examples/conf/druid/single-server/large/_common/log4j2.xml new file mode 100644 index 00000000000..bd2b3a687cc --- /dev/null +++ b/examples/conf/druid/single-server/large/_common/log4j2.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + diff --git a/examples/conf/druid/single-server/large/broker/jvm.config b/examples/conf/druid/single-server/large/broker/jvm.config new file mode 100644 index 00000000000..da8c305bdb3 --- /dev/null +++ b/examples/conf/druid/single-server/large/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms16g +-Xmx16g +-XX:MaxDirectMemorySize=8g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/large/broker/main.config b/examples/conf/druid/single-server/large/broker/main.config new file mode 100644 index 00000000000..9bc9a6ff397 --- /dev/null +++ b/examples/conf/druid/single-server/large/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/examples/conf/druid/single-server/large/broker/runtime.properties b/examples/conf/druid/single-server/large/broker/runtime.properties new file mode 100644 index 00000000000..a38e324b052 --- /dev/null +++ b/examples/conf/druid/single-server/large/broker/runtime.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=5000000 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=8 +druid.processing.numThreads=1 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/examples/conf/druid/single-server/large/coordinator-overlord/jvm.config b/examples/conf/druid/single-server/large/coordinator-overlord/jvm.config new file mode 100644 index 00000000000..04b4729e66b --- /dev/null +++ b/examples/conf/druid/single-server/large/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms24g +-Xmx24g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/examples/conf/druid/single-server/large/coordinator-overlord/main.config b/examples/conf/druid/single-server/large/coordinator-overlord/main.config new file mode 100644 index 00000000000..a74101f6eb1 --- /dev/null +++ b/examples/conf/druid/single-server/large/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties new file mode 100644 index 00000000000..8928cc9f8ed --- /dev/null +++ b/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.runner.type=remote +druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/large/historical/jvm.config b/examples/conf/druid/single-server/large/historical/jvm.config new file mode 100644 index 00000000000..bd616d11d06 --- /dev/null +++ b/examples/conf/druid/single-server/large/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms16g +-Xmx16g +-XX:MaxDirectMemorySize=32g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/large/historical/main.config b/examples/conf/druid/single-server/large/historical/main.config new file mode 100644 index 00000000000..42be713ad5d --- /dev/null +++ b/examples/conf/druid/single-server/large/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/examples/conf/druid/single-server/large/historical/runtime.properties b/examples/conf/druid/single-server/large/historical/runtime.properties new file mode 100644 index 00000000000..dcb0004a2f9 --- /dev/null +++ b/examples/conf/druid/single-server/large/historical/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=8 +druid.processing.numThreads=32 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}] +druid.server.maxSize=300000000000 + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=1000000000 diff --git a/examples/quickstart/tutorial/conf/druid/broker/jvm.config b/examples/conf/druid/single-server/large/middleManager/jvm.config similarity index 86% rename from examples/quickstart/tutorial/conf/druid/broker/jvm.config rename to examples/conf/druid/single-server/large/middleManager/jvm.config index 076a2226185..82597e0b956 100644 --- a/examples/quickstart/tutorial/conf/druid/broker/jvm.config +++ b/examples/conf/druid/single-server/large/middleManager/jvm.config @@ -1,8 +1,8 @@ -server -Xms256m -Xmx256m --XX:MaxDirectMemorySize=768m -XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp diff --git a/examples/conf/druid/single-server/large/middleManager/main.config b/examples/conf/druid/single-server/large/middleManager/main.config new file mode 100644 index 00000000000..0e627338085 --- /dev/null +++ b/examples/conf/druid/single-server/large/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/examples/conf/druid/single-server/large/middleManager/runtime.properties b/examples/conf/druid/single-server/large/middleManager/runtime.properties new file mode 100644 index 00000000000..54b462f6840 --- /dev/null +++ b/examples/conf/druid/single-server/large/middleManager/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=8 + +# Task launch parameters +druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100000000 +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/single-server/large/router/jvm.config b/examples/conf/druid/single-server/large/router/jvm.config new file mode 100644 index 00000000000..a844c40b1ea --- /dev/null +++ b/examples/conf/druid/single-server/large/router/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms1g +-Xmx1g +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/large/router/main.config b/examples/conf/druid/single-server/large/router/main.config new file mode 100644 index 00000000000..4194b69c259 --- /dev/null +++ b/examples/conf/druid/single-server/large/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/examples/quickstart/tutorial/conf/druid/router/runtime.properties b/examples/conf/druid/single-server/large/router/runtime.properties similarity index 100% rename from examples/quickstart/tutorial/conf/druid/router/runtime.properties rename to examples/conf/druid/single-server/large/router/runtime.properties diff --git a/examples/conf/druid/single-server/medium/_common/common.runtime.properties b/examples/conf/druid/single-server/medium/_common/common.runtime.properties new file mode 100644 index 00000000000..6dd696f1f02 --- /dev/null +++ b/examples/conf/druid/single-server/medium/_common/common.runtime.properties @@ -0,0 +1,145 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: http://druid.io/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password"] + + +# +# SQL +# +druid.sql.enable=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false \ No newline at end of file diff --git a/examples/conf/druid/single-server/medium/_common/log4j2.xml b/examples/conf/druid/single-server/medium/_common/log4j2.xml new file mode 100644 index 00000000000..bd2b3a687cc --- /dev/null +++ b/examples/conf/druid/single-server/medium/_common/log4j2.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + diff --git a/examples/conf/druid/single-server/medium/broker/jvm.config b/examples/conf/druid/single-server/medium/broker/jvm.config new file mode 100644 index 00000000000..bdb24117693 --- /dev/null +++ b/examples/conf/druid/single-server/medium/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms8g +-Xmx8g +-XX:MaxDirectMemorySize=16g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/medium/broker/main.config b/examples/conf/druid/single-server/medium/broker/main.config new file mode 100644 index 00000000000..9bc9a6ff397 --- /dev/null +++ b/examples/conf/druid/single-server/medium/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/examples/conf/druid/single-server/medium/broker/runtime.properties b/examples/conf/druid/single-server/medium/broker/runtime.properties new file mode 100644 index 00000000000..17e881490d5 --- /dev/null +++ b/examples/conf/druid/single-server/medium/broker/runtime.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=5000000 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=4 +druid.processing.numThreads=1 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/examples/conf/druid/single-server/medium/coordinator-overlord/jvm.config b/examples/conf/druid/single-server/medium/coordinator-overlord/jvm.config new file mode 100644 index 00000000000..38d2e1ebbee --- /dev/null +++ b/examples/conf/druid/single-server/medium/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms12g +-Xmx12g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/examples/conf/druid/single-server/medium/coordinator-overlord/main.config b/examples/conf/druid/single-server/medium/coordinator-overlord/main.config new file mode 100644 index 00000000000..a74101f6eb1 --- /dev/null +++ b/examples/conf/druid/single-server/medium/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties new file mode 100644 index 00000000000..8928cc9f8ed --- /dev/null +++ b/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.runner.type=remote +druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/medium/historical/jvm.config b/examples/conf/druid/single-server/medium/historical/jvm.config new file mode 100644 index 00000000000..f31ae92ba9f --- /dev/null +++ b/examples/conf/druid/single-server/medium/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms8g +-Xmx8g +-XX:MaxDirectMemorySize=13g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/medium/historical/main.config b/examples/conf/druid/single-server/medium/historical/main.config new file mode 100644 index 00000000000..42be713ad5d --- /dev/null +++ b/examples/conf/druid/single-server/medium/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/examples/conf/druid/single-server/medium/historical/runtime.properties b/examples/conf/druid/single-server/medium/historical/runtime.properties new file mode 100644 index 00000000000..1a70a71fce2 --- /dev/null +++ b/examples/conf/druid/single-server/medium/historical/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=4 +druid.processing.numThreads=16 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}] +druid.server.maxSize=300000000000 + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=256000000 diff --git a/examples/quickstart/tutorial/conf/druid/historical/jvm.config b/examples/conf/druid/single-server/medium/middleManager/jvm.config similarity index 86% rename from examples/quickstart/tutorial/conf/druid/historical/jvm.config rename to examples/conf/druid/single-server/medium/middleManager/jvm.config index 076a2226185..82597e0b956 100644 --- a/examples/quickstart/tutorial/conf/druid/historical/jvm.config +++ b/examples/conf/druid/single-server/medium/middleManager/jvm.config @@ -1,8 +1,8 @@ -server -Xms256m -Xmx256m --XX:MaxDirectMemorySize=768m -XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp diff --git a/examples/conf/druid/single-server/medium/middleManager/main.config b/examples/conf/druid/single-server/medium/middleManager/main.config new file mode 100644 index 00000000000..0e627338085 --- /dev/null +++ b/examples/conf/druid/single-server/medium/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/examples/conf/druid/single-server/medium/middleManager/runtime.properties b/examples/conf/druid/single-server/medium/middleManager/runtime.properties new file mode 100644 index 00000000000..55d9f1cbb29 --- /dev/null +++ b/examples/conf/druid/single-server/medium/middleManager/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=4 + +# Task launch parameters +druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100000000 +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/router/jvm.config b/examples/conf/druid/single-server/medium/router/jvm.config similarity index 87% rename from examples/conf/druid/router/jvm.config rename to examples/conf/druid/single-server/medium/router/jvm.config index 37b200e628f..6fc9b4cdfef 100644 --- a/examples/conf/druid/router/jvm.config +++ b/examples/conf/druid/single-server/medium/router/jvm.config @@ -2,7 +2,7 @@ -Xms512m -Xmx512m -XX:+UseG1GC --XX:MaxDirectMemorySize=512m +-XX:MaxDirectMemorySize=128m -XX:+ExitOnOutOfMemoryError -Duser.timezone=UTC -Dfile.encoding=UTF-8 diff --git a/examples/conf/druid/single-server/medium/router/main.config b/examples/conf/druid/single-server/medium/router/main.config new file mode 100644 index 00000000000..4194b69c259 --- /dev/null +++ b/examples/conf/druid/single-server/medium/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/examples/conf/druid/single-server/medium/router/runtime.properties b/examples/conf/druid/single-server/medium/router/runtime.properties new file mode 100644 index 00000000000..497d3b4d3aa --- /dev/null +++ b/examples/conf/druid/single-server/medium/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties b/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties new file mode 100644 index 00000000000..6dd696f1f02 --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties @@ -0,0 +1,145 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: http://druid.io/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password"] + + +# +# SQL +# +druid.sql.enable=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false \ No newline at end of file diff --git a/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml b/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml new file mode 100644 index 00000000000..bd2b3a687cc --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + diff --git a/examples/conf/druid/single-server/micro-quickstart/broker/jvm.config b/examples/conf/druid/single-server/micro-quickstart/broker/jvm.config new file mode 100644 index 00000000000..cf72e5d2df9 --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms512m +-Xmx512m +-XX:MaxDirectMemorySize=768m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/micro-quickstart/broker/main.config b/examples/conf/druid/single-server/micro-quickstart/broker/main.config new file mode 100644 index 00000000000..9bc9a6ff397 --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/examples/quickstart/tutorial/conf/druid/broker/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/broker/runtime.properties similarity index 100% rename from examples/quickstart/tutorial/conf/druid/broker/runtime.properties rename to examples/conf/druid/single-server/micro-quickstart/broker/runtime.properties diff --git a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/jvm.config b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/jvm.config new file mode 100644 index 00000000000..8ab5708c320 --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms256m +-Xmx256m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/main.config b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/main.config new file mode 100644 index 00000000000..a74101f6eb1 --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties new file mode 100644 index 00000000000..8928cc9f8ed --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.runner.type=remote +druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/micro-quickstart/historical/jvm.config b/examples/conf/druid/single-server/micro-quickstart/historical/jvm.config new file mode 100644 index 00000000000..8747402e85a --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms512m +-Xmx512m +-XX:MaxDirectMemorySize=1280m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/micro-quickstart/historical/main.config b/examples/conf/druid/single-server/micro-quickstart/historical/main.config new file mode 100644 index 00000000000..42be713ad5d --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/examples/quickstart/tutorial/conf/druid/historical/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/historical/runtime.properties similarity index 94% rename from examples/quickstart/tutorial/conf/druid/historical/runtime.properties rename to examples/conf/druid/single-server/micro-quickstart/historical/runtime.properties index dc78b0f762a..594abafb7b3 100644 --- a/examples/quickstart/tutorial/conf/druid/historical/runtime.properties +++ b/examples/conf/druid/single-server/micro-quickstart/historical/runtime.properties @@ -24,8 +24,8 @@ druid.plaintextPort=8083 druid.server.http.numThreads=12 # Processing threads and buffers -druid.processing.buffer.sizeBytes=100000000 -druid.processing.numMergeBuffers=1 +druid.processing.buffer.sizeBytes=200000000 +druid.processing.numMergeBuffers=2 druid.processing.numThreads=2 druid.processing.tmpDir=var/druid/processing diff --git a/examples/quickstart/tutorial/conf/druid/middleManager/jvm.config b/examples/conf/druid/single-server/micro-quickstart/middleManager/jvm.config similarity index 93% rename from examples/quickstart/tutorial/conf/druid/middleManager/jvm.config rename to examples/conf/druid/single-server/micro-quickstart/middleManager/jvm.config index 8a650fb358c..30b023cfb8d 100644 --- a/examples/quickstart/tutorial/conf/druid/middleManager/jvm.config +++ b/examples/conf/druid/single-server/micro-quickstart/middleManager/jvm.config @@ -2,6 +2,7 @@ -Xms64m -Xmx64m -XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp diff --git a/examples/conf/druid/single-server/micro-quickstart/middleManager/main.config b/examples/conf/druid/single-server/micro-quickstart/middleManager/main.config new file mode 100644 index 00000000000..0e627338085 --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties new file mode 100644 index 00000000000..8be6e568dc6 --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=2 + +# Task launch parameters +druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=12 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100000000 +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/quickstart/tutorial/conf/druid/router/jvm.config b/examples/conf/druid/single-server/micro-quickstart/router/jvm.config similarity index 100% rename from examples/quickstart/tutorial/conf/druid/router/jvm.config rename to examples/conf/druid/single-server/micro-quickstart/router/jvm.config diff --git a/examples/conf/druid/single-server/micro-quickstart/router/main.config b/examples/conf/druid/single-server/micro-quickstart/router/main.config new file mode 100644 index 00000000000..4194b69c259 --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/examples/conf/druid/single-server/micro-quickstart/router/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/router/runtime.properties new file mode 100644 index 00000000000..497d3b4d3aa --- /dev/null +++ b/examples/conf/druid/single-server/micro-quickstart/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/examples/conf/druid/single-server/small/_common/common.runtime.properties b/examples/conf/druid/single-server/small/_common/common.runtime.properties new file mode 100644 index 00000000000..6dd696f1f02 --- /dev/null +++ b/examples/conf/druid/single-server/small/_common/common.runtime.properties @@ -0,0 +1,145 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: http://druid.io/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password"] + + +# +# SQL +# +druid.sql.enable=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false \ No newline at end of file diff --git a/examples/conf/druid/single-server/small/_common/log4j2.xml b/examples/conf/druid/single-server/small/_common/log4j2.xml new file mode 100644 index 00000000000..bd2b3a687cc --- /dev/null +++ b/examples/conf/druid/single-server/small/_common/log4j2.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + diff --git a/examples/conf/druid/single-server/small/broker/jvm.config b/examples/conf/druid/single-server/small/broker/jvm.config new file mode 100644 index 00000000000..1017bce2ab5 --- /dev/null +++ b/examples/conf/druid/single-server/small/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms4g +-Xmx4g +-XX:MaxDirectMemorySize=3g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/small/broker/main.config b/examples/conf/druid/single-server/small/broker/main.config new file mode 100644 index 00000000000..9bc9a6ff397 --- /dev/null +++ b/examples/conf/druid/single-server/small/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/examples/conf/druid/broker/runtime.properties b/examples/conf/druid/single-server/small/broker/runtime.properties similarity index 85% rename from examples/conf/druid/broker/runtime.properties rename to examples/conf/druid/single-server/small/broker/runtime.properties index 9421053d09d..072c9e9b859 100644 --- a/examples/conf/druid/broker/runtime.properties +++ b/examples/conf/druid/single-server/small/broker/runtime.properties @@ -21,15 +21,17 @@ druid.service=druid/broker druid.plaintextPort=8082 # HTTP server settings -druid.server.http.numThreads=60 +druid.server.http.numThreads=50 # HTTP client settings -druid.broker.http.numConnections=10 +druid.broker.http.numConnections=40 +druid.broker.http.maxQueuedBytes=5000000 # Processing threads and buffers -druid.processing.buffer.sizeBytes=536870912 +druid.processing.buffer.sizeBytes=500000000 druid.processing.numMergeBuffers=2 druid.processing.numThreads=1 +druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead druid.broker.cache.useCache=false diff --git a/examples/quickstart/tutorial/conf/druid/coordinator/jvm.config b/examples/conf/druid/single-server/small/coordinator-overlord/jvm.config similarity index 88% rename from examples/quickstart/tutorial/conf/druid/coordinator/jvm.config rename to examples/conf/druid/single-server/small/coordinator-overlord/jvm.config index 162faa0a2f0..c853ea840d7 100644 --- a/examples/quickstart/tutorial/conf/druid/coordinator/jvm.config +++ b/examples/conf/druid/single-server/small/coordinator-overlord/jvm.config @@ -1,7 +1,8 @@ -server --Xms128m --Xmx128m +-Xms6g +-Xmx6g -XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp diff --git a/examples/conf/druid/single-server/small/coordinator-overlord/main.config b/examples/conf/druid/single-server/small/coordinator-overlord/main.config new file mode 100644 index 00000000000..a74101f6eb1 --- /dev/null +++ b/examples/conf/druid/single-server/small/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties new file mode 100644 index 00000000000..8928cc9f8ed --- /dev/null +++ b/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.runner.type=remote +druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/small/historical/jvm.config b/examples/conf/druid/single-server/small/historical/jvm.config new file mode 100644 index 00000000000..21906078471 --- /dev/null +++ b/examples/conf/druid/single-server/small/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms4g +-Xmx4g +-XX:MaxDirectMemorySize=8g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/small/historical/main.config b/examples/conf/druid/single-server/small/historical/main.config new file mode 100644 index 00000000000..42be713ad5d --- /dev/null +++ b/examples/conf/druid/single-server/small/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/examples/conf/druid/single-server/small/historical/runtime.properties b/examples/conf/druid/single-server/small/historical/runtime.properties new file mode 100644 index 00000000000..144a029c4dd --- /dev/null +++ b/examples/conf/druid/single-server/small/historical/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=50 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=2 +druid.processing.numThreads=8 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}] +druid.server.maxSize=300000000000 + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=256000000 diff --git a/examples/conf/druid/middleManager/jvm.config b/examples/conf/druid/single-server/small/middleManager/jvm.config similarity index 84% rename from examples/conf/druid/middleManager/jvm.config rename to examples/conf/druid/single-server/small/middleManager/jvm.config index 8a650fb358c..a5862186437 100644 --- a/examples/conf/druid/middleManager/jvm.config +++ b/examples/conf/druid/single-server/small/middleManager/jvm.config @@ -1,7 +1,8 @@ -server --Xms64m --Xmx64m +-Xms128m +-Xmx128m -XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp diff --git a/examples/conf/druid/single-server/small/middleManager/main.config b/examples/conf/druid/single-server/small/middleManager/main.config new file mode 100644 index 00000000000..0e627338085 --- /dev/null +++ b/examples/conf/druid/single-server/small/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/examples/quickstart/tutorial/conf/druid/middleManager/runtime.properties b/examples/conf/druid/single-server/small/middleManager/runtime.properties similarity index 85% rename from examples/quickstart/tutorial/conf/druid/middleManager/runtime.properties rename to examples/conf/druid/single-server/small/middleManager/runtime.properties index 2262d8473bb..1665e46f9f1 100644 --- a/examples/quickstart/tutorial/conf/druid/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/small/middleManager/runtime.properties @@ -28,11 +28,12 @@ druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -Duser.timezone=UTC -Dfile.e druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads -druid.server.http.numThreads=9 +druid.server.http.numThreads=50 # Processing threads and buffers on Peons -druid.indexer.fork.property.druid.processing.buffer.sizeBytes=201326592 -druid.indexer.fork.property.druid.processing.numThreads=2 +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100000000 +druid.indexer.fork.property.druid.processing.numThreads=1 # Hadoop indexing druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/single-server/small/router/jvm.config b/examples/conf/druid/single-server/small/router/jvm.config new file mode 100644 index 00000000000..ad8a52de508 --- /dev/null +++ b/examples/conf/druid/single-server/small/router/jvm.config @@ -0,0 +1,11 @@ +-server +-Xms512m +-Xmx512m +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/small/router/main.config b/examples/conf/druid/single-server/small/router/main.config new file mode 100644 index 00000000000..4194b69c259 --- /dev/null +++ b/examples/conf/druid/single-server/small/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/examples/conf/druid/single-server/small/router/runtime.properties b/examples/conf/druid/single-server/small/router/runtime.properties new file mode 100644 index 00000000000..497d3b4d3aa --- /dev/null +++ b/examples/conf/druid/single-server/small/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/examples/conf/druid/single-server/xlarge/_common/common.runtime.properties b/examples/conf/druid/single-server/xlarge/_common/common.runtime.properties new file mode 100644 index 00000000000..6dd696f1f02 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/_common/common.runtime.properties @@ -0,0 +1,145 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: http://druid.io/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password"] + + +# +# SQL +# +druid.sql.enable=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false \ No newline at end of file diff --git a/examples/conf/druid/single-server/xlarge/_common/log4j2.xml b/examples/conf/druid/single-server/xlarge/_common/log4j2.xml new file mode 100644 index 00000000000..bd2b3a687cc --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/_common/log4j2.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + diff --git a/examples/conf/druid/single-server/xlarge/broker/jvm.config b/examples/conf/druid/single-server/xlarge/broker/jvm.config new file mode 100644 index 00000000000..a8844b275eb --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms24g +-Xmx24g +-XX:MaxDirectMemorySize=12g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/xlarge/broker/main.config b/examples/conf/druid/single-server/xlarge/broker/main.config new file mode 100644 index 00000000000..9bc9a6ff397 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/examples/conf/druid/single-server/xlarge/broker/runtime.properties b/examples/conf/druid/single-server/xlarge/broker/runtime.properties new file mode 100644 index 00000000000..6d4b3699fd8 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/broker/runtime.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=5000000 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=16 +druid.processing.numThreads=1 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/examples/conf/druid/single-server/xlarge/coordinator-overlord/jvm.config b/examples/conf/druid/single-server/xlarge/coordinator-overlord/jvm.config new file mode 100644 index 00000000000..04b4729e66b --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms24g +-Xmx24g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/examples/conf/druid/single-server/xlarge/coordinator-overlord/main.config b/examples/conf/druid/single-server/xlarge/coordinator-overlord/main.config new file mode 100644 index 00000000000..a74101f6eb1 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties new file mode 100644 index 00000000000..8928cc9f8ed --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.runner.type=remote +druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/xlarge/historical/jvm.config b/examples/conf/druid/single-server/xlarge/historical/jvm.config new file mode 100644 index 00000000000..92553d3c8b4 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms24g +-Xmx24g +-XX:MaxDirectMemorySize=44g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/xlarge/historical/main.config b/examples/conf/druid/single-server/xlarge/historical/main.config new file mode 100644 index 00000000000..42be713ad5d --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/examples/conf/druid/single-server/xlarge/historical/runtime.properties b/examples/conf/druid/single-server/xlarge/historical/runtime.properties new file mode 100644 index 00000000000..11856c5a2e0 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/historical/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500000000 +druid.processing.numMergeBuffers=16 +druid.processing.numThreads=64 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}] +druid.server.maxSize=300000000000 + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=2000000000 diff --git a/examples/conf/druid/single-server/xlarge/middleManager/jvm.config b/examples/conf/druid/single-server/xlarge/middleManager/jvm.config new file mode 100644 index 00000000000..82597e0b956 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/middleManager/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms256m +-Xmx256m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/xlarge/middleManager/main.config b/examples/conf/druid/single-server/xlarge/middleManager/main.config new file mode 100644 index 00000000000..0e627338085 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties new file mode 100644 index 00000000000..889d20deb18 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=16 + +# Task launch parameters +druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=500000000 +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/single-server/xlarge/router/jvm.config b/examples/conf/druid/single-server/xlarge/router/jvm.config new file mode 100644 index 00000000000..a844c40b1ea --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/router/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms1g +-Xmx1g +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/xlarge/router/main.config b/examples/conf/druid/single-server/xlarge/router/main.config new file mode 100644 index 00000000000..4194b69c259 --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/examples/conf/druid/single-server/xlarge/router/runtime.properties b/examples/conf/druid/single-server/xlarge/router/runtime.properties new file mode 100644 index 00000000000..497d3b4d3aa --- /dev/null +++ b/examples/conf/druid/single-server/xlarge/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/examples/conf/supervise/cluster/data.conf b/examples/conf/supervise/cluster/data.conf new file mode 100644 index 00000000000..3047288de80 --- /dev/null +++ b/examples/conf/supervise/cluster/data.conf @@ -0,0 +1,11 @@ +:verify bin/verify-java +:verify bin/verify-version-check + +historical bin/run-druid historical conf/druid/cluster/data/historical +middleManager bin/run-druid middleManager conf/druid/cluster/data/middleManager + +# Uncomment to use Tranquility Server +#!p95 tranquility-server bin/tranquility server -configFile conf/tranquility/server.json + +# Uncomment to use Tranquility Kafka +#!p95 tranquility-kafka bin/tranquility kafka -configFile conf/tranquility/kafka.json diff --git a/examples/conf/supervise/cluster/master-no-zk.conf b/examples/conf/supervise/cluster/master-no-zk.conf new file mode 100644 index 00000000000..8b22448f24a --- /dev/null +++ b/examples/conf/supervise/cluster/master-no-zk.conf @@ -0,0 +1,5 @@ +:verify bin/verify-java +:verify bin/verify-version-check + +coordinator bin/run-druid coordinator conf/druid/cluster/data/coordinator +!p80 overlord bin/run-druid overlord conf/druid/cluster/data/overlord diff --git a/examples/conf/supervise/cluster/master-with-zk.conf b/examples/conf/supervise/cluster/master-with-zk.conf new file mode 100644 index 00000000000..8eeea0cff8a --- /dev/null +++ b/examples/conf/supervise/cluster/master-with-zk.conf @@ -0,0 +1,6 @@ +:verify bin/verify-java +:verify bin/verify-version-check + +!p10 zk bin/run-zk conf +coordinator bin/run-druid coordinator conf/druid/cluster/data/coordinator +!p80 overlord bin/run-druid overlord conf/druid/cluster/data/overlord diff --git a/examples/conf/supervise/cluster/query.conf b/examples/conf/supervise/cluster/query.conf new file mode 100644 index 00000000000..cd6ec376be9 --- /dev/null +++ b/examples/conf/supervise/cluster/query.conf @@ -0,0 +1,5 @@ +:verify bin/verify-java +:verify bin/verify-version-check + +broker bin/run-druid broker conf/druid/cluster/data/broker +router bin/run-druid router conf/druid/cluster/data/router diff --git a/examples/conf/supervise/single-server/large.conf b/examples/conf/supervise/single-server/large.conf new file mode 100644 index 00000000000..27007bad42d --- /dev/null +++ b/examples/conf/supervise/single-server/large.conf @@ -0,0 +1,13 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/large +broker bin/run-druid broker conf/druid/single-server/large +router bin/run-druid router conf/druid/single-server/large +historical bin/run-druid historical conf/druid/single-server/large +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/large + +# Uncomment to use Tranquility Server +#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/medium.conf b/examples/conf/supervise/single-server/medium.conf new file mode 100644 index 00000000000..73fcb4d694a --- /dev/null +++ b/examples/conf/supervise/single-server/medium.conf @@ -0,0 +1,13 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/medium +broker bin/run-druid broker conf/druid/single-server/medium +router bin/run-druid router conf/druid/single-server/medium +historical bin/run-druid historical conf/druid/single-server/medium +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/medium + +# Uncomment to use Tranquility Server +#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/micro-quickstart.conf b/examples/conf/supervise/single-server/micro-quickstart.conf new file mode 100644 index 00000000000..abe79f88ab0 --- /dev/null +++ b/examples/conf/supervise/single-server/micro-quickstart.conf @@ -0,0 +1,13 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/micro-quickstart +broker bin/run-druid broker conf/druid/single-server/micro-quickstart +router bin/run-druid router conf/druid/single-server/micro-quickstart +historical bin/run-druid historical conf/druid/single-server/micro-quickstart +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/micro-quickstart + +# Uncomment to use Tranquility Server +#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/small.conf b/examples/conf/supervise/single-server/small.conf new file mode 100644 index 00000000000..13b987335a0 --- /dev/null +++ b/examples/conf/supervise/single-server/small.conf @@ -0,0 +1,13 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/small +broker bin/run-druid broker conf/druid/single-server/small +router bin/run-druid router conf/druid/single-server/small +historical bin/run-druid historical conf/druid/single-server/small +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/small + +# Uncomment to use Tranquility Server +#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/xlarge.conf b/examples/conf/supervise/single-server/xlarge.conf new file mode 100644 index 00000000000..343b30b1318 --- /dev/null +++ b/examples/conf/supervise/single-server/xlarge.conf @@ -0,0 +1,13 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/xlarge +broker bin/run-druid broker conf/druid/single-server/xlarge +router bin/run-druid router conf/druid/single-server/xlarge +historical bin/run-druid historical conf/druid/single-server/xlarge +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/xlarge + +# Uncomment to use Tranquility Server +#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/quickstart/tutorial/conf/tranquility/wikipedia-server.json b/examples/conf/tranquility/wikipedia-server.json similarity index 100% rename from examples/quickstart/tutorial/conf/tranquility/wikipedia-server.json rename to examples/conf/tranquility/wikipedia-server.json diff --git a/examples/quickstart/tutorial/conf/zk/jvm.config b/examples/conf/zk/jvm.config similarity index 100% rename from examples/quickstart/tutorial/conf/zk/jvm.config rename to examples/conf/zk/jvm.config diff --git a/examples/quickstart/tutorial/conf/zk/log4j.xml b/examples/conf/zk/log4j.xml similarity index 100% rename from examples/quickstart/tutorial/conf/zk/log4j.xml rename to examples/conf/zk/log4j.xml diff --git a/examples/quickstart/tutorial/conf/zk/zoo.cfg b/examples/conf/zk/zoo.cfg similarity index 100% rename from examples/quickstart/tutorial/conf/zk/zoo.cfg rename to examples/conf/zk/zoo.cfg diff --git a/examples/quickstart/tutorial/conf/tutorial-cluster.conf b/examples/quickstart/tutorial/conf/tutorial-cluster.conf deleted file mode 100644 index 4c2c70de77e..00000000000 --- a/examples/quickstart/tutorial/conf/tutorial-cluster.conf +++ /dev/null @@ -1,14 +0,0 @@ -:verify bin/verify-java -:verify bin/verify-default-ports -:kill-timeout 10 - -!p10 zk bin/run-zk quickstart/tutorial/conf -coordinator bin/run-druid coordinator quickstart/tutorial/conf -broker bin/run-druid broker quickstart/tutorial/conf -router bin/run-druid router quickstart/tutorial/conf -historical bin/run-druid historical quickstart/tutorial/conf -!p80 overlord bin/run-druid overlord quickstart/tutorial/conf -!p90 middleManager bin/run-druid middleManager quickstart/tutorial/conf - -# Uncomment to use Tranquility Server -#!p95 tranquility-server tranquility/bin/tranquility server -configFile quickstart/tutorial/conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/pom.xml b/pom.xml index 2218f49ed29..a0fff228faa 100644 --- a/pom.xml +++ b/pom.xml @@ -1536,7 +1536,7 @@ publications/** codestyle/*-forbidden-apis.txt - quickstart/tutorial/conf/** + conf/** docker/*.conf target/** licenses/**