diff --git a/docs/configuration/index.md b/docs/configuration/index.md index f5930585303..b92e71dcdee 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -31,7 +31,7 @@ A recommended way of organizing Druid configuration files can be seen in the `co ``` $ ls -R conf -druid tranquility +druid conf/druid: _common broker coordinator historical middleManager overlord @@ -53,9 +53,6 @@ jvm.config runtime.properties conf/druid/overlord: jvm.config runtime.properties - -conf/tranquility: -kafka.json server.json ``` Each directory has a `runtime.properties` file containing configuration properties for the specific Druid process corresponding to the directory (e.g., `historical`). diff --git a/docs/development/extensions-contrib/thrift.md b/docs/development/extensions-contrib/thrift.md index 68497c11a5d..051c6e9a5fa 100644 --- a/docs/development/extensions-contrib/thrift.md +++ b/docs/development/extensions-contrib/thrift.md @@ -43,47 +43,7 @@ If you plan to read LZO-compressed Thrift files, you will need to download versi | thriftJar | String | path of thrift jar, if not provided, it will try to find the thrift class in classpath. Thrift jar in batch ingestion should be uploaded to HDFS first and configure `jobProperties` with `"tmpjars":"/path/to/your/thrift.jar"` | no | | thriftClass | String | classname of thrift | yes | -- Realtime Ingestion (tranquility example) - -```json -{ - "dataSources": [{ - "spec": { - "dataSchema": { - "dataSource": "book", - "granularitySpec": { }, - "parser": { - "type": "thrift", - "thriftClass": "org.apache.druid.data.input.thrift.Book", - "protocol": "compact", - "parseSpec": { - "format": "json", - ... - } - }, - "metricsSpec": [...] - }, - "tuningConfig": {...} - }, - "properties": {...} - }], - "properties": {...} -} -``` - -To use it with tranquility, - -```bash -bin/tranquility kafka \ - -configFile $jsonConfig \ - -Ddruid.extensions.directory=/path/to/extensions \ - -Ddruid.extensions.loadList='["druid-thrift-extensions"]' -``` - -Hadoop-client is also needed, you may copy all the hadoop-client dependency jars into directory `druid-thrift-extensions` to make is simple. - - -- Batch Ingestion - `inputFormat` and `tmpjars` should be set. +- Batch Ingestion example - `inputFormat` and `tmpjars` should be set. This is for batch ingestion using the HadoopDruidIndexer. The inputFormat of inputSpec in ioConfig could be one of `"org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"` and `com.twitter.elephantbird.mapreduce.input.LzoThriftBlockInputFormat`. Be careful, when `LzoThriftBlockInputFormat` is used, thrift class must be provided twice. diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 37c17055cdb..482d7389c7b 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -26,9 +26,8 @@ sidebar_label: "Apache Kafka" The Kafka indexing service enables the configuration of *supervisors* on the Overlord, which facilitate ingestion from Kafka by managing the creation and lifetime of Kafka indexing tasks. These indexing tasks read events using Kafka's own -partition and offset mechanism and are therefore able to provide guarantees of exactly-once ingestion. They are also -able to read non-recent events from Kafka and are not subject to the window period considerations imposed on other -ingestion mechanisms using Tranquility. The supervisor oversees the state of the indexing tasks to coordinate handoffs, +partition and offset mechanism and are therefore able to provide guarantees of exactly-once ingestion. +The supervisor oversees the state of the indexing tasks to coordinate handoffs, manage failures, and ensure that the scalability and replication requirements are maintained. This service is provided in the `druid-kafka-indexing-service` core Apache Druid (incubating) extension (see diff --git a/docs/development/extensions-core/kinesis-ingestion.md b/docs/development/extensions-core/kinesis-ingestion.md index b5fe83706d7..aec4699543e 100644 --- a/docs/development/extensions-core/kinesis-ingestion.md +++ b/docs/development/extensions-core/kinesis-ingestion.md @@ -26,9 +26,8 @@ sidebar_label: "Amazon Kinesis" Similar to the [Kafka indexing service](./kafka-ingestion.md), the Kinesis indexing service enables the configuration of *supervisors* on the Overlord, which facilitate ingestion from Kinesis by managing the creation and lifetime of Kinesis indexing tasks. These indexing tasks read events using Kinesis's own -Shards and Sequence Number mechanism and are therefore able to provide guarantees of exactly-once ingestion. They are also -able to read non-recent events from Kinesis and are not subject to the window period considerations imposed on other -ingestion mechanisms using Tranquility. The supervisor oversees the state of the indexing tasks to coordinate handoffs, manage failures, +Shards and Sequence Number mechanism and are therefore able to provide guarantees of exactly-once ingestion. +The supervisor oversees the state of the indexing tasks to coordinate handoffs, manage failures, and ensure that the scalability and replication requirements are maintained. The Kinesis indexing service is provided as the `druid-kinesis-indexing-service` core Apache Druid (incubating) extension (see diff --git a/docs/development/extensions-core/protobuf.md b/docs/development/extensions-core/protobuf.md index b4afaf0c45a..0882d6839ff 100644 --- a/docs/development/extensions-core/protobuf.md +++ b/docs/development/extensions-core/protobuf.md @@ -41,7 +41,7 @@ This example demonstrates how to load Protobuf messages from Kafka. Please read Files used in this example are found at `./examples/quickstart/protobuf` in your Druid directory. -- We will use [Kafka Indexing Service](./kafka-ingestion.md) instead of Tranquility. +- We will use [Kafka Indexing Service](./kafka-ingestion.md). - Kafka broker host is `localhost:9092`. - Kafka topic is `metrics_pb` instead of `metrics`. - datasource name is `metrics-kafka-pb` instead of `metrics-kafka` to avoid the confusion. diff --git a/docs/ingestion/tasks.md b/docs/ingestion/tasks.md index d658d0816a9..fa120a74ad4 100644 --- a/docs/ingestion/tasks.md +++ b/docs/ingestion/tasks.md @@ -294,7 +294,7 @@ Once `forceTimeChunkLock` is unset, the task will choose a proper lock type to u Please note that segment lock is not always available. The most common use case where time chunk lock is enforced is when an overwriting task changes the segment granularity. Also, the segment locking is supported by only native indexing tasks and Kafka/Kinesis indexing tasks. -The Hadoop indexing tasks and realtime indexing tasks (with [Tranquility](tranquility.md)) don't support it yet. +Hadoop indexing tasks and `index_realtime` tasks (used by [Tranquility](tranquility.md)) don't support it yet. `forceTimeChunkLock` in the task context is only applied to individual tasks. If you want to unset it for all tasks, you would want to set `druid.indexer.tasklock.forceTimeChunkLock` to false in the [overlord configuration](../configuration/index.html#overlord-operations). diff --git a/docs/querying/multitenancy.md b/docs/querying/multitenancy.md index 342c112897c..645cc924f2c 100644 --- a/docs/querying/multitenancy.md +++ b/docs/querying/multitenancy.md @@ -60,13 +60,8 @@ With batch indexing, you can use [single-dimension partitioning](../ingestion/ha to partition your data by tenant_id. Druid always partitions by time first, but the secondary partition within each time bucket will be on tenant_id. -With realtime indexing, you have a couple of options. - -1. Partition on tenant_id upfront. You'd do this by tweaking the stream you send to Druid. If you're using Kafka then -you can have your Kafka producer partition your topic by a hash of tenant_id. If you're using Tranquility then you can -define a custom [Partitioner](http://static.druid.io/tranquility/api/latest/#com.metamx.tranquility.partition.Partitioner). -2. Reindex your older data periodically. You can do this with the ["dataSource" input spec](../ingestion/hadoop.html#datasource). -You can use this in concert with single-dimension partitioning to repartition your data. +With realtime indexing, you'd do this by tweaking the stream you send to Druid. For example, if you're using Kafka then +you can have your Kafka producer partition your topic by a hash of tenant_id. ## Customizing data distribution diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md index ad3cb3c2bf4..be6db4a5f5a 100644 --- a/docs/tutorials/cluster.md +++ b/docs/tutorials/cluster.md @@ -405,9 +405,6 @@ inbound connections on the following: - 8082 (Broker) - 8088 (Router, if used) -### Other -- 8200 (Tranquility Server, if used) - > In production, we recommend deploying ZooKeeper and your metadata store on their own dedicated hardware, > rather than on the Master server. @@ -455,24 +452,6 @@ You can add more Data servers as needed. > For clusters with complex resource allocation needs, you can break apart Historicals and MiddleManagers and scale the components individually. > This also allows you take advantage of Druid's built-in MiddleManager autoscaling facility. -### Tranquility - -If you are doing push-based stream ingestion with Kafka or over HTTP, you can also start Tranquility Server on the Data server. - -For large scale production, Data server processes and the Tranquility Server can still be co-located. - -If you are running Tranquility (not server) with a stream processor, you can co-locate Tranquility with the stream processor and not require Tranquility Server. - -First install Tranquility: - -```bash -curl http://static.druid.io/tranquility/releases/tranquility-distribution-0.8.3.tgz -o tranquility-distribution-0.8.3.tgz -tar -xzf tranquility-distribution-0.8.3.tgz -mv tranquility-distribution-0.8.3 tranquility -``` - -Afterwards, in `conf/supervise/cluster/data.conf`, uncomment out the `tranquility-server` line, and restart the Data server processes. - ## Start Query Server Copy the Druid distribution and your edited configurations to your Query servers. diff --git a/examples/conf/supervise/cluster/data.conf b/examples/conf/supervise/cluster/data.conf index 32f61d96064..f2a16e6cf33 100644 --- a/examples/conf/supervise/cluster/data.conf +++ b/examples/conf/supervise/cluster/data.conf @@ -2,6 +2,3 @@ historical bin/run-druid historical conf/druid/cluster/data middleManager bin/run-druid middleManager conf/druid/cluster/data - -# Uncomment to use Tranquility Server -#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/large.conf b/examples/conf/supervise/single-server/large.conf index 27007bad42d..2f7baf6fa5b 100644 --- a/examples/conf/supervise/single-server/large.conf +++ b/examples/conf/supervise/single-server/large.conf @@ -8,6 +8,3 @@ broker bin/run-druid broker conf/druid/single-server/large router bin/run-druid router conf/druid/single-server/large historical bin/run-druid historical conf/druid/single-server/large !p90 middleManager bin/run-druid middleManager conf/druid/single-server/large - -# Uncomment to use Tranquility Server -#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/medium.conf b/examples/conf/supervise/single-server/medium.conf index 73fcb4d694a..84d15caba9b 100644 --- a/examples/conf/supervise/single-server/medium.conf +++ b/examples/conf/supervise/single-server/medium.conf @@ -8,6 +8,3 @@ broker bin/run-druid broker conf/druid/single-server/medium router bin/run-druid router conf/druid/single-server/medium historical bin/run-druid historical conf/druid/single-server/medium !p90 middleManager bin/run-druid middleManager conf/druid/single-server/medium - -# Uncomment to use Tranquility Server -#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/micro-quickstart.conf b/examples/conf/supervise/single-server/micro-quickstart.conf index abe79f88ab0..de1c0f8d68f 100644 --- a/examples/conf/supervise/single-server/micro-quickstart.conf +++ b/examples/conf/supervise/single-server/micro-quickstart.conf @@ -8,6 +8,3 @@ broker bin/run-druid broker conf/druid/single-server/micro-quickstart router bin/run-druid router conf/druid/single-server/micro-quickstart historical bin/run-druid historical conf/druid/single-server/micro-quickstart !p90 middleManager bin/run-druid middleManager conf/druid/single-server/micro-quickstart - -# Uncomment to use Tranquility Server -#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/nano-quickstart.conf b/examples/conf/supervise/single-server/nano-quickstart.conf index 9465cf48eee..8baf7698021 100644 --- a/examples/conf/supervise/single-server/nano-quickstart.conf +++ b/examples/conf/supervise/single-server/nano-quickstart.conf @@ -8,6 +8,3 @@ broker bin/run-druid broker conf/druid/single-server/nano-quickstart router bin/run-druid router conf/druid/single-server/nano-quickstart historical bin/run-druid historical conf/druid/single-server/nano-quickstart !p90 middleManager bin/run-druid middleManager conf/druid/single-server/nano-quickstart - -# Uncomment to use Tranquility Server -#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/small.conf b/examples/conf/supervise/single-server/small.conf index 13b987335a0..ef4cad54da4 100644 --- a/examples/conf/supervise/single-server/small.conf +++ b/examples/conf/supervise/single-server/small.conf @@ -8,6 +8,3 @@ broker bin/run-druid broker conf/druid/single-server/small router bin/run-druid router conf/druid/single-server/small historical bin/run-druid historical conf/druid/single-server/small !p90 middleManager bin/run-druid middleManager conf/druid/single-server/small - -# Uncomment to use Tranquility Server -#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/supervise/single-server/xlarge.conf b/examples/conf/supervise/single-server/xlarge.conf index 343b30b1318..dbec43f4e77 100644 --- a/examples/conf/supervise/single-server/xlarge.conf +++ b/examples/conf/supervise/single-server/xlarge.conf @@ -8,6 +8,3 @@ broker bin/run-druid broker conf/druid/single-server/xlarge router bin/run-druid router conf/druid/single-server/xlarge historical bin/run-druid historical conf/druid/single-server/xlarge !p90 middleManager bin/run-druid middleManager conf/druid/single-server/xlarge - -# Uncomment to use Tranquility Server -#!p95 tranquility-server tranquility/bin/tranquility server -configFile conf/tranquility/wikipedia-server.json -Ddruid.extensions.loadList=[] diff --git a/examples/conf/tranquility/kafka.json b/examples/conf/tranquility/kafka.json deleted file mode 100644 index fb7c9aeb4ca..00000000000 --- a/examples/conf/tranquility/kafka.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "dataSources" : { - "metrics-kafka" : { - "spec" : { - "dataSchema" : { - "dataSource" : "metrics-kafka", - "parser" : { - "type" : "string", - "parseSpec" : { - "timestampSpec" : { - "column" : "timestamp", - "format" : "auto" - }, - "dimensionsSpec" : { - "dimensions" : [], - "dimensionExclusions" : [ - "timestamp", - "value" - ] - }, - "format" : "json" - } - }, - "granularitySpec" : { - "type" : "uniform", - "segmentGranularity" : "hour", - "queryGranularity" : "none" - }, - "metricsSpec" : [ - { - "type" : "count", - "name" : "count" - }, - { - "name" : "value_sum", - "type" : "doubleSum", - "fieldName" : "value" - }, - { - "fieldName" : "value", - "name" : "value_min", - "type" : "doubleMin" - }, - { - "type" : "doubleMax", - "name" : "value_max", - "fieldName" : "value" - } - ] - }, - "ioConfig" : { - "type" : "realtime" - }, - "tuningConfig" : { - "type" : "realtime", - "intermediatePersistPeriod" : "PT10M", - "windowPeriod" : "PT10M" - } - }, - "properties" : { - "task.partitions" : "1", - "task.replicants" : "1", - "topicPattern" : "metrics" - } - } - }, - "properties" : { - "zookeeper.connect" : "master.example.com", - "druid.discovery.curator.path" : "/druid/discovery", - "druid.selectors.indexing.serviceName" : "druid/overlord", - "commit.periodMillis" : "15000", - "consumer.numThreads" : "2", - "kafka.zookeeper.connect" : "kafka-zk.example.com", - "kafka.group.id" : "tranquility-kafka" - } -} diff --git a/examples/conf/tranquility/server.json b/examples/conf/tranquility/server.json deleted file mode 100644 index ff810f72fbb..00000000000 --- a/examples/conf/tranquility/server.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataSources" : { - "metrics" : { - "spec" : { - "dataSchema" : { - "dataSource" : "metrics", - "parser" : { - "type" : "string", - "parseSpec" : { - "timestampSpec" : { - "column" : "timestamp", - "format" : "auto" - }, - "dimensionsSpec" : { - "dimensions" : [], - "dimensionExclusions" : [ - "timestamp", - "value" - ] - }, - "format" : "json" - } - }, - "granularitySpec" : { - "type" : "uniform", - "segmentGranularity" : "hour", - "queryGranularity" : "none" - }, - "metricsSpec" : [ - { - "type" : "count", - "name" : "count" - }, - { - "name" : "value_sum", - "type" : "doubleSum", - "fieldName" : "value" - }, - { - "fieldName" : "value", - "name" : "value_min", - "type" : "doubleMin" - }, - { - "type" : "doubleMax", - "name" : "value_max", - "fieldName" : "value" - } - ] - }, - "ioConfig" : { - "type" : "realtime" - }, - "tuningConfig" : { - "type" : "realtime", - "intermediatePersistPeriod" : "PT10M", - "windowPeriod" : "PT10M" - } - }, - "properties" : { - "task.partitions" : "1", - "task.replicants" : "1" - } - } - }, - "properties" : { - "zookeeper.connect" : "master.example.com", - "druid.discovery.curator.path" : "/druid/discovery", - "druid.selectors.indexing.serviceName" : "druid/overlord", - "http.port" : "8200", - "http.threads" : "8" - } -} diff --git a/examples/conf/tranquility/wikipedia-server.json b/examples/conf/tranquility/wikipedia-server.json deleted file mode 100644 index 0895cab1e62..00000000000 --- a/examples/conf/tranquility/wikipedia-server.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "dataSources" : { - "wikipedia" : { - "spec" : { - "dataSchema" : { - "dataSource" : "wikipedia", - "parser" : { - "type" : "string", - "parseSpec" : { - "format" : "json", - "dimensionsSpec" : { - "dimensions" : [ - "channel", - "cityName", - "comment", - "countryIsoCode", - "countryName", - "isAnonymous", - "isMinor", - "isNew", - "isRobot", - "isUnpatrolled", - "metroCode", - "namespace", - "page", - "regionIsoCode", - "regionName", - "user" - ] - }, - "timestampSpec": { - "column": "time", - "format": "iso" - } - } - }, - "metricsSpec" : [ - { - "name" : "added", - "type" : "longSum", - "fieldName" : "added" - }, - { - "name" : "deleted", - "type" : "longSum", - "fieldName" : "deleted" - }, - { - "name" : "delta", - "type" : "longSum", - "fieldName" : "delta" - } - ], - "granularitySpec" : { - "type" : "uniform", - "segmentGranularity" : "day", - "queryGranularity" : "none", - "intervals" : ["2015-09-12/2015-09-13"], - "rollup" : false - } - }, - "ioConfig" : { - "type" : "realtime" - }, - "tuningConfig" : { - "type" : "realtime", - "intermediatePersistPeriod" : "PT10M", - "windowPeriod" : "P3650D" - } - }, - "properties" : { - "task.partitions" : "1", - "task.replicants" : "1" - } - } - }, - "properties" : { - "zookeeper.connect" : "localhost", - "druid.discovery.curator.path" : "/druid/discovery", - "druid.selectors.indexing.serviceName" : "druid/overlord", - "http.port" : "8200", - "http.threads" : "9" - } -}