From 0851f2206c129f3dabdd87b48b7531cdb5ba2e37 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 21 Mar 2018 18:19:27 -0700 Subject: [PATCH] Expanded documentation for DataSketches aggregators. (#5513) Originally written by @AlexanderSaydakov in druid-io/druid-io.github.io#448. I also added redirects and updated links to point to the new datasketches-extension.html landing page for the extension, rather than to the old page about theta sketches. --- docs/_redirects.json | 3 +- .../extensions-core/datasketches-extension.md | 19 +++ .../extensions-core/datasketches-quantiles.md | 92 +++++++++++ ...s-aggregators.md => datasketches-theta.md} | 10 +- .../extensions-core/datasketches-tuple.md | 155 ++++++++++++++++++ docs/content/development/extensions.md | 2 +- docs/content/querying/aggregations.md | 2 +- docs/content/querying/sql.md | 2 +- docs/content/toc.md | 2 +- 9 files changed, 277 insertions(+), 10 deletions(-) create mode 100644 docs/content/development/extensions-core/datasketches-extension.md create mode 100644 docs/content/development/extensions-core/datasketches-quantiles.md rename docs/content/development/extensions-core/{datasketches-aggregators.md => datasketches-theta.md} (91%) create mode 100644 docs/content/development/extensions-core/datasketches-tuple.md diff --git a/docs/_redirects.json b/docs/_redirects.json index 0e6ef6ff986..b0e61992779 100644 --- a/docs/_redirects.json +++ b/docs/_redirects.json @@ -94,7 +94,8 @@ {"source": "configuration/simple-cluster.html", "target": "../tutorials/cluster.html"}, {"source": "design/concepts-and-terminology.html", "target": "index.html"}, {"source": "development/approximate-histograms.html", "target": "extensions-core/approximate-histograms.html"}, - {"source": "development/datasketches-aggregators.html", "target": "extensions-core/datasketches-aggregators.html"}, + {"source": "development/datasketches-aggregators.html", "target": "extensions-core/datasketches-extension.html"}, + {"source": "development/extensions-core/datasketches-aggregators.html", "target": "datasketches-extension.html"}, {"source": "development/libraries.html", "target": "/libraries.html"}, {"source": "development/kafka-simple-consumer-firehose.html", "target": "extensions-contrib/kafka-simple.html"}, {"source": "development/select-query.html", "target": "../querying/select-query.html"}, diff --git a/docs/content/development/extensions-core/datasketches-extension.md b/docs/content/development/extensions-core/datasketches-extension.md new file mode 100644 index 00000000000..a8a1d8738b8 --- /dev/null +++ b/docs/content/development/extensions-core/datasketches-extension.md @@ -0,0 +1,19 @@ +--- +layout: doc_page +--- + +## DataSketches extension + +Druid aggregators based on [datasketches](http://datasketches.github.io/) library. Sketches are data structures implementing approximate streaming mergeable algorithms. Sketches can be ingested from the outside of Druid or built from raw data at ingestion time. Sketches can be stored in Druid segments as additive metrics. + +To use the datasketch aggregators, make sure you [include](../../operations/including-extensions.html) the extension in your config file: + +``` +druid.extensions.loadList=["druid-datasketches"] +``` + +The following aggregators are available: + +1. [Theta sketch](datasketches-theta.html), useful for approximate set counting, and supporting union, intersection, and difference operations. +2. [Quantiles sketch](datasketches-quantiles.html). +3. [Tuple sketch](datasketches-tuple.html). diff --git a/docs/content/development/extensions-core/datasketches-quantiles.md b/docs/content/development/extensions-core/datasketches-quantiles.md new file mode 100644 index 00000000000..c9eeb84c177 --- /dev/null +++ b/docs/content/development/extensions-core/datasketches-quantiles.md @@ -0,0 +1,92 @@ +--- +layout: doc_page +--- + +## DataSketches Quantiles Sketch module + +This module provides Druid aggregators based on numeric quantiles DoublesSketch from [datasketches](http://datasketches.github.io/) library. Quantiles sketch is a mergeable streaming algorithm to estimate the distribution of values, and approximately answer queries about the rank of a value, probability mass function of the distribution (PMF) or histogram, cummulative distribution function (CDF), and quantiles (median, min, max, 95th percentile and such). See [Quantiles Sketch Overview](https://datasketches.github.io/docs/Quantiles/QuantilesOverview.html). + +There are three major modes of operation: + +1. Ingesting sketches built outside of Druid (say, with Pig or Hive) +2. Building sketches from raw data during ingestion +3. Building sketches from raw data at query time + +To use this aggregator, make sure you [include](../../operations/including-extensions.html) the extension in your config file: + +``` +druid.extensions.loadList=["druid-datasketches"] +``` + +### Aggregator + +The result of the aggregation is a DoublesSketch that is the union of all sketches either built from raw data or read from the segments. + +```json +{ + "type" : "quantilesDoublesSketch", + "name" : , + "fieldName" : , + "k": + } +``` + +|property|description|required?| +|--------|-----------|---------| +|type|This String should always be "quantilesDoublesSketch"|yes| +|name|A String for the output (result) name of the calculation.|yes| +|fieldName|A String for the name of the input field (can contain sketches or raw numeric values).|yes| +|k|Parameter that determines the accuracy and size of the sketch. Higher k means higher accuracy but more space to store sketches. Must be a power of 2 from 2 to 32768. See the [Quantiles Accuracy](https://datasketches.github.io/docs/Quantiles/QuantilesAccuracy.html) for details. |no, defaults to 128| + +### Post Aggregators + +#### Quantile + +This returns an approximation to the value that would be preceded by a given fraction of a hypothetical sorted version of the input stream. + +```json +{ + "type" : "quantilesDoublesSketchToQuantile", + "name": , + "field" : , + "fraction" : +} +``` + +#### Quantiles + +This returns an array of quantiles corresponding to a given array of fractions + +```json +{ + "type" : "quantilesDoublesSketchToQuantiles", + "name": , + "field" : , + "fractions" : +} +``` + +#### Histogram + +This returns an approximation to the histogram given an array of split points that define the histogram bins. An array of m unique, monotonically increasing split points divide the real number line into m+1 consecutive disjoint intervals. The definition of an interval is inclusive of the left split point and exclusive of the right split point. + +```json +{ + "type" : "quantilesDoublesSketchToHistogram", + "name": , + "field" : , + "splitPoints" : +} +``` + +#### Sketch Summary + +This returns a summary of the sketch that can be used for debugging. This is the result of calling toString() method. + +```json +{ + "type" : "quantilesDoublesSketchToString", + "name": , + "field" : +} +``` diff --git a/docs/content/development/extensions-core/datasketches-aggregators.md b/docs/content/development/extensions-core/datasketches-theta.md similarity index 91% rename from docs/content/development/extensions-core/datasketches-aggregators.md rename to docs/content/development/extensions-core/datasketches-theta.md index bf7e4a601f3..07d179555be 100644 --- a/docs/content/development/extensions-core/datasketches-aggregators.md +++ b/docs/content/development/extensions-core/datasketches-theta.md @@ -2,13 +2,13 @@ layout: doc_page --- -## DataSketches aggregator +## DataSketches Theta Sketch module -Druid aggregators based on [datasketches](http://datasketches.github.io/) library. Note that sketch algorithms are approximate; see details in the "Accuracy" section of the datasketches doc. -At ingestion time, this aggregator creates the theta sketch objects which get stored in Druid segments. Logically speaking, a theta sketch object can be thought of as a Set data structure. At query time, sketches are read and aggregated (set unioned) together. In the end, by default, you receive the estimate of the number of unique entries in the sketch object. Also, you can use post aggregators to do union, intersection or difference on sketch columns in the same row. -Note that you can use `thetaSketch` aggregator on columns which were not ingested using same, it will return estimated cardinality of the column. It is recommended to use it at ingestion time as well to make querying faster. +This module provides Druid aggregators based on Theta sketch from [datasketches](http://datasketches.github.io/) library. Note that sketch algorithms are approximate; see details in the "Accuracy" section of the datasketches doc. +At ingestion time, this aggregator creates the Theta sketch objects which get stored in Druid segments. Logically speaking, a Theta sketch object can be thought of as a Set data structure. At query time, sketches are read and aggregated (set unioned) together. In the end, by default, you receive the estimate of the number of unique entries in the sketch object. Also, you can use post aggregators to do union, intersection or difference on sketch columns in the same row. +Note that you can use `thetaSketch` aggregator on columns which were not ingested using the same. It will return estimated cardinality of the column. It is recommended to use it at ingestion time as well to make querying faster. -To use the datasketch aggregators, make sure you [include](../../operations/including-extensions.html) the extension in your config file: +To use this aggregator, make sure you [include](../../operations/including-extensions.html) the extension in your config file: ``` druid.extensions.loadList=["druid-datasketches"] diff --git a/docs/content/development/extensions-core/datasketches-tuple.md b/docs/content/development/extensions-core/datasketches-tuple.md new file mode 100644 index 00000000000..37ed8ead400 --- /dev/null +++ b/docs/content/development/extensions-core/datasketches-tuple.md @@ -0,0 +1,155 @@ +--- +layout: doc_page +--- + +## DataSketches Tuple Sketch module + +This module provides Druid aggregators based on Tuple sketch from [datasketches](http://datasketches.github.io/) library. ArrayOfDoublesSketch sketches extend the functionality of the count-distinct Theta sketches by adding arrays of double values associated with unique keys. + +To use this aggregator, make sure you [include](../../operations/including-extensions.html) the extension in your config file: + +``` +druid.extensions.loadList=["druid-datasketches"] +``` + +### Aggregators + +```json +{ + "type" : "arrayOfDoublesSketch", + "name" : , + "fieldName" : , + "nominalEntries": , + "numberOfValues" : , + "metricColumns" : + } +``` + +|property|description|required?| +|--------|-----------|---------| +|type|This String should always be "arrayOfDoublesSketch"|yes| +|name|A String for the output (result) name of the calculation.|yes| +|fieldName|A String for the name of the input field.|yes| +|nominalEntries|Parameter that determines the accuracy and size of the sketch. Higher k means higher accuracy but more space to store sketches. Must be a power of 2. See the [Theta sketch accuracy](https://datasketches.github.io/docs/Theta/ThetaErrorTable.html) for details. |no, defaults to 16384| +|numberOfValues|Number of values associated with each distinct key. |no, defaults to 1| +|metricCoulumns|If building sketches from raw data, an array of names of the input columns containing numeric vaues to be associated with each distinct key.|no, defaults to empty array| + +### Post Aggregators + +#### Estimate of the number of distinct keys + +Returns a distinct count estimate from a given ArrayOfDoublesSketch. + +```json +{ + "type" : "arrayOfDoublesSketchToEstimate", + "name": , + "field" : +} +``` + +#### Estimate of the number of distinct keys with error bounds + +Returns a distinct count estimate and error bounds from a given ArrayOfDoublesSketch. The result will be three double values: estimate of the number of distinct keys, lower bound and upper bound. The bounds are provided at the given number of standard deviations (optional, defaults to 1). This must be an integer value of 1, 2 or 3 corresponding to approximately 68.3%, 95.4% and 99.7% confidence intervals. + +```json +{ + "type" : "arrayOfDoublesSketchToEstimateAndBounds", + "name": , + "field" : , + "numStdDevs", +} +``` + +#### Number of retained entries + +Returns the number of retained entries from a given ArrayOfDoublesSketch. + +```json +{ + "type" : "arrayOfDoublesSketchToNumEntries", + "name": , + "field" : +} +``` + +#### Mean values for each column + +Returns a list of mean values from a given ArrayOfDoublesSketch. The result will be N double values, where N is the number of double values kept in the sketch per key. + +```json +{ + "type" : "arrayOfDoublesSketchToMeans", + "name": , + "field" : +} +``` + +#### Variance values for each column + +Returns a list of variance values from a given ArrayOfDoublesSketch. The result will be N double values, where N is the number of double values kept in the sketch per key. + +```json +{ + "type" : "arrayOfDoublesSketchToVariances", + "name": , + "field" : +} +``` + +#### Quantiles sketch from a column + +Returns a quanitles DoublesSketch constructed from a given column of values from a given ArrayOfDoublesSketch using optional parameter k that determines the accuracy and size of the quantiles sketch. See [Quantiles Sketch Module](datasketches-quantiles.html) + +* The column number is 1-based and is optional (the default is 1). +* The parameter k is optional (the default is defined in the sketch library). +* The result is a quantiles sketch. + +```json +{ + "type" : "arrayOfDoublesSketchToQuantilesSketch", + "name": , + "field" : , + "column" : , + "k" : +} +``` + +#### Set Operations + +Returns a result of a specified set operation on the given array of sketches. Supported operations are: union, intersection and set difference (UNION, INTERSECT, NOT). + +```json +{ + "type" : "arrayOfDoublesSketchSetOp", + "name": , + "func": <"UNION"|"INTERSECT"|"NOT">, + "fields" : , + "nominalEntries" : , + "numberOfValues" : +} +``` + +#### Student's t-test + +Performs Student's t-test and returns a list of p-values given two instances of ArrayOfDoublesSketch. The result will be N double values, where N is the number of double values kept in the sketch per key. See [t-test documentation](http://commons.apache.org/proper/commons-math/javadocs/api-3.4/org/apache/commons/math3/stat/inference/TTest.html). + +```json +{ + "type" : "arrayOfDoublesSketchTTest", + "name": , + "fields" : , +} +``` + +#### Sketch summary + +Returns a human-readable summary of a given ArrayOfDoublesSketch. This is a string returned by toString() method of the sketch. This can be useful for debugging. + +```json +{ + "type" : "arrayOfDoublesSketchToString", + "name": , + "field" : +} +``` diff --git a/docs/content/development/extensions.md b/docs/content/development/extensions.md index 4e2bea613fa..41c725ec91b 100644 --- a/docs/content/development/extensions.md +++ b/docs/content/development/extensions.md @@ -24,7 +24,7 @@ Core extensions are maintained by Druid committers. |druid-avro-extensions|Support for data in Apache Avro data format.|[link](../development/extensions-core/avro.html)| |druid-basic-security|Support for Basic HTTP authentication and role-based access control.|[link](../development/extensions-core/druid-basic-security.html)| |druid-caffeine-cache|A local cache implementation backed by Caffeine.|[link](../development/extensions-core/caffeine-cache.html)| -|druid-datasketches|Support for approximate counts and set operations with [DataSketches](http://datasketches.github.io/).|[link](../development/extensions-core/datasketches-aggregators.html)| +|druid-datasketches|Support for approximate counts and set operations with [DataSketches](http://datasketches.github.io/).|[link](../development/extensions-core/datasketches-extension.html)| |druid-hdfs-storage|HDFS deep storage.|[link](../development/extensions-core/hdfs.html)| |druid-histogram|Approximate histograms and quantiles aggregator.|[link](../development/extensions-core/approximate-histograms.html)| |druid-kafka-eight|Kafka ingest firehose (high level consumer) for realtime nodes.|[link](../development/extensions-core/kafka-eight-firehose.html)| diff --git a/docs/content/querying/aggregations.md b/docs/content/querying/aggregations.md index 420daf97afb..b0ce5cc24c9 100644 --- a/docs/content/querying/aggregations.md +++ b/docs/content/querying/aggregations.md @@ -332,7 +332,7 @@ The HyperLogLog algorithm generates decimal estimates with some error. "round" c values to whole numbers. Note that even with rounding, the cardinality is still an estimate. The "round" field only affects query-time behavior, and is ignored at ingestion-time. -For more approximate aggregators, please see [theta sketches](../development/extensions-core/datasketches-aggregators.html). +For more approximate aggregators, check out the [DataSketches extension](../development/extensions-core/datasketches-extension.html). ## Miscellaneous Aggregations diff --git a/docs/content/querying/sql.md b/docs/content/querying/sql.md index 5796a66701c..a993a76f953 100644 --- a/docs/content/querying/sql.md +++ b/docs/content/querying/sql.md @@ -220,7 +220,7 @@ Druid does not support all SQL features, including: Additionally, some Druid features are not supported by the SQL language. Some unsupported Druid features include: - [Multi-value dimensions](multi-value-dimensions.html). -- [DataSketches aggregators](../development/extensions-core/datasketches-aggregators.html). +- [DataSketches aggregators](../development/extensions-core/datasketches-extension.html). - [Spatial filters](../development/geo.html). - [Query cancellation](querying.html#query-cancellation). diff --git a/docs/content/toc.md b/docs/content/toc.md index e51f95c05d4..addd5ebdc83 100644 --- a/docs/content/toc.md +++ b/docs/content/toc.md @@ -103,7 +103,7 @@ layout: toc * Experimental Features * [Overview](/docs/VERSION/development/experimental.html) * [Approximate Histograms and Quantiles](/docs/VERSION/development/extensions-core/approximate-histograms.html) - * [Datasketches](/docs/VERSION/development/extensions-core/datasketches-aggregators.html) + * [Datasketches](/docs/VERSION/development/extensions-core/datasketches-extension.html) * [Geographic Queries](/docs/VERSION/development/geo.html) * [Router](/docs/VERSION/development/router.html) * [Kafka Indexing Service](/docs/VERSION/development/extensions-core/kafka-ingestion.html)