mirror of https://github.com/apache/druid.git
Web console: Support all possible metric types in the data loader (#8785)
* Support all possible metric types in the data loader * added more sketches
This commit is contained in:
Vadim Ogievetsky
Fangjin Yang
parent
a95e3d438e
commit
929a8b6337
|
|
@ -59,6 +59,7 @@ exports[`retention dialog matches snapshot 1`] = `
|
|||
|
||||
<a
|
||||
href="https://druid.apache.org/docs/0.16.0-incubating/operations/rule-configuration.html"
|
||||
rel="noopener noreferrer"
|
||||
target="_blank"
|
||||
>
|
||||
documentation
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import { IconNames } from '@blueprintjs/icons';
|
|||
import axios from 'axios';
|
||||
import React from 'react';
|
||||
|
||||
import { RuleEditor } from '../../components';
|
||||
import { ExternalLink, RuleEditor } from '../../components';
|
||||
import { QueryManager } from '../../utils';
|
||||
import { DRUID_DOCS_VERSION } from '../../variables';
|
||||
import { SnitchDialog } from '../snitch-dialog/snitch-dialog';
|
||||
|
|
@ -183,12 +183,11 @@ export class RetentionDialog extends React.PureComponent<
|
|||
<p>
|
||||
Druid uses rules to determine what data should be retained in the cluster. The rules are
|
||||
evaluated in order from top to bottom. For more information please refer to the{' '}
|
||||
<a
|
||||
<ExternalLink
|
||||
href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/operations/rule-configuration.html`}
|
||||
target="_blank"
|
||||
>
|
||||
documentation
|
||||
</a>
|
||||
</ExternalLink>
|
||||
.
|
||||
</p>
|
||||
<FormGroup>{(currentRules || []).map(this.renderRule)}</FormGroup>
|
||||
|
|
|
|||
|
|
@ -668,6 +668,7 @@ const METRIC_SPEC_FORM_FIELDS: Field<MetricSpec>[] = [
|
|||
{
|
||||
name: 'name',
|
||||
type: 'string',
|
||||
info: <>The metric name as it will appear in Druid.</>,
|
||||
},
|
||||
{
|
||||
name: 'type',
|
||||
|
|
@ -694,37 +695,24 @@ const METRIC_SPEC_FORM_FIELDS: Field<MetricSpec>[] = [
|
|||
group: 'last',
|
||||
suggestions: ['longLast', 'doubleLast', 'floatLast'],
|
||||
},
|
||||
'cardinality',
|
||||
'thetaSketch',
|
||||
{
|
||||
group: 'HLLSketch',
|
||||
suggestions: ['HLLSketchBuild', 'HLLSketchMerge'],
|
||||
},
|
||||
'quantilesDoublesSketch',
|
||||
'momentSketch',
|
||||
'fixedBucketsHistogram',
|
||||
'hyperUnique',
|
||||
'filtered',
|
||||
],
|
||||
info: <>The aggregation function to apply.</>,
|
||||
},
|
||||
{
|
||||
name: 'fieldName',
|
||||
type: 'string',
|
||||
defined: m => {
|
||||
return [
|
||||
'longSum',
|
||||
'doubleSum',
|
||||
'floatSum',
|
||||
'longMin',
|
||||
'doubleMin',
|
||||
'floatMin',
|
||||
'longMax',
|
||||
'doubleMax',
|
||||
'floatMax',
|
||||
'longFirst',
|
||||
'doubleFirst',
|
||||
'floatFirst',
|
||||
'stringFirst',
|
||||
'longLast',
|
||||
'doubleLast',
|
||||
'floatLast',
|
||||
'stringLast',
|
||||
'cardinality',
|
||||
'hyperUnique',
|
||||
].includes(m.type);
|
||||
},
|
||||
defined: m => m.type !== 'filtered',
|
||||
info: <>The column name for the aggregator to operate on.</>,
|
||||
},
|
||||
{
|
||||
name: 'maxStringBytes',
|
||||
|
|
@ -742,21 +730,196 @@ const METRIC_SPEC_FORM_FIELDS: Field<MetricSpec>[] = [
|
|||
return ['stringFirst', 'stringLast'].includes(m.type);
|
||||
},
|
||||
},
|
||||
// filtered
|
||||
{
|
||||
name: 'filter',
|
||||
type: 'json',
|
||||
defined: m => {
|
||||
return m.type === 'filtered';
|
||||
},
|
||||
defined: m => m.type === 'filtered',
|
||||
},
|
||||
{
|
||||
name: 'aggregator',
|
||||
type: 'json',
|
||||
defined: m => {
|
||||
return m.type === 'filtered';
|
||||
defined: m => m.type === 'filtered',
|
||||
},
|
||||
// thetaSketch
|
||||
{
|
||||
name: 'size',
|
||||
type: 'number',
|
||||
defined: m => m.type === 'thetaSketch',
|
||||
defaultValue: 16384,
|
||||
info: (
|
||||
<>
|
||||
<p>
|
||||
Must be a power of 2. Internally, size refers to the maximum number of entries sketch
|
||||
object will retain. Higher size means higher accuracy but more space to store sketches.
|
||||
Note that after you index with a particular size, druid will persist sketch in segments
|
||||
and you will use size greater or equal to that at query time.
|
||||
</p>
|
||||
<p>
|
||||
See the{' '}
|
||||
<ExternalLink href="https://datasketches.github.io/docs/Theta/ThetaSize.html">
|
||||
DataSketches site
|
||||
</ExternalLink>{' '}
|
||||
for details.
|
||||
</p>
|
||||
<p>In general, We recommend just sticking to default size.</p>
|
||||
</>
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'isInputThetaSketch',
|
||||
type: 'boolean',
|
||||
defined: m => m.type === 'thetaSketch',
|
||||
defaultValue: false,
|
||||
info: (
|
||||
<>
|
||||
This should only be used at indexing time if your input data contains theta sketch objects.
|
||||
This would be the case if you use datasketches library outside of Druid, say with Pig/Hive,
|
||||
to produce the data that you are ingesting into Druid
|
||||
</>
|
||||
),
|
||||
},
|
||||
// HLLSketchBuild & HLLSketchMerge
|
||||
{
|
||||
name: 'lgK',
|
||||
type: 'number',
|
||||
defined: m => m.type === 'HLLSketchBuild' || m.type === 'HLLSketchMerge',
|
||||
defaultValue: 12,
|
||||
info: (
|
||||
<>
|
||||
<p>
|
||||
log2 of K that is the number of buckets in the sketch, parameter that controls the size
|
||||
and the accuracy.
|
||||
</p>
|
||||
<p>Must be between 4 to 21 inclusively.</p>
|
||||
</>
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'tgtHllType',
|
||||
type: 'string',
|
||||
defined: m => m.type === 'HLLSketchBuild' || m.type === 'HLLSketchMerge',
|
||||
defaultValue: 'HLL_4',
|
||||
suggestions: ['HLL_4', 'HLL_6', 'HLL_8'],
|
||||
info: (
|
||||
<>
|
||||
The type of the target HLL sketch. Must be <Code>HLL_4</Code>, <Code>HLL_6</Code>, or{' '}
|
||||
<Code>HLL_8</Code>.
|
||||
</>
|
||||
),
|
||||
},
|
||||
// quantilesDoublesSketch
|
||||
{
|
||||
name: 'k',
|
||||
type: 'number',
|
||||
defined: m => m.type === 'quantilesDoublesSketch',
|
||||
defaultValue: 128,
|
||||
info: (
|
||||
<>
|
||||
<p>
|
||||
Parameter that determines the accuracy and size of the sketch. Higher k means higher
|
||||
accuracy but more space to store sketches.
|
||||
</p>
|
||||
<p>
|
||||
Must be a power of 2 from 2 to 32768. See the{' '}
|
||||
<ExternalLink href="https://datasketches.github.io/docs/Quantiles/QuantilesAccuracy.html">
|
||||
Quantiles Accuracy
|
||||
</ExternalLink>{' '}
|
||||
for details.
|
||||
</p>
|
||||
</>
|
||||
),
|
||||
},
|
||||
// momentSketch
|
||||
{
|
||||
name: 'k',
|
||||
type: 'number',
|
||||
defined: m => m.type === 'momentSketch',
|
||||
required: true,
|
||||
info: (
|
||||
<>
|
||||
Parameter that determines the accuracy and size of the sketch. Higher k means higher
|
||||
accuracy but more space to store sketches. Usable range is generally [3,15]
|
||||
</>
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'compress',
|
||||
type: 'boolean',
|
||||
defined: m => m.type === 'momentSketch',
|
||||
defaultValue: true,
|
||||
info: (
|
||||
<>
|
||||
Flag for whether the aggregator compresses numeric values using arcsinh. Can improve
|
||||
robustness to skewed and long-tailed distributions, but reduces accuracy slightly on more
|
||||
uniform distributions.
|
||||
</>
|
||||
),
|
||||
},
|
||||
// fixedBucketsHistogram
|
||||
{
|
||||
name: 'lowerLimit',
|
||||
type: 'number',
|
||||
defined: m => m.type === 'fixedBucketsHistogram',
|
||||
required: true,
|
||||
info: <>Lower limit of the histogram.</>,
|
||||
},
|
||||
{
|
||||
name: 'upperLimit',
|
||||
type: 'number',
|
||||
defined: m => m.type === 'fixedBucketsHistogram',
|
||||
required: true,
|
||||
info: <>Upper limit of the histogram.</>,
|
||||
},
|
||||
{
|
||||
name: 'numBuckets',
|
||||
type: 'number',
|
||||
defined: m => m.type === 'fixedBucketsHistogram',
|
||||
defaultValue: 10,
|
||||
required: true,
|
||||
info: (
|
||||
<>
|
||||
Number of buckets for the histogram. The range <Code>[lowerLimit, upperLimit]</Code> will be
|
||||
divided into <Code>numBuckets</Code> intervals of equal size.
|
||||
</>
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'outlierHandlingMode',
|
||||
type: 'string',
|
||||
defined: m => m.type === 'fixedBucketsHistogram',
|
||||
required: true,
|
||||
suggestions: ['ignore', 'overflow', 'clip'],
|
||||
info: (
|
||||
<>
|
||||
<p>
|
||||
Specifies how values outside of <Code>[lowerLimit, upperLimit]</Code> will be handled.
|
||||
</p>
|
||||
<p>
|
||||
Supported modes are <Code>ignore</Code>, <Code>overflow</Code>, and <Code>clip</Code>. See
|
||||
<ExternalLink
|
||||
href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-core/approximate-histograms.html#outlier-handling-modes`}
|
||||
>
|
||||
outlier handling modes
|
||||
</ExternalLink>{' '}
|
||||
for more details.
|
||||
</p>
|
||||
</>
|
||||
),
|
||||
},
|
||||
// hyperUnique
|
||||
{
|
||||
name: 'isInputHyperUnique',
|
||||
type: 'boolean',
|
||||
defined: m => m.type === 'hyperUnique',
|
||||
defaultValue: false,
|
||||
info: (
|
||||
<>
|
||||
This can be set to true to index precomputed HLL (Base64 encoded output from druid-hll is
|
||||
expected).
|
||||
</>
|
||||
),
|
||||
},
|
||||
// ToDo: fill in approximates
|
||||
];
|
||||
|
||||
export function getMetricSpecFormFields() {
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@
|
|||
bottom: 0;
|
||||
left: 0;
|
||||
content: '';
|
||||
border: 2px solid #008adc;
|
||||
border: 2px solid #48aff0;
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
|
|
@ -195,7 +195,7 @@
|
|||
left: 0;
|
||||
right: 0;
|
||||
content: '';
|
||||
border: 2px solid #008adc;
|
||||
border: 2px solid #ff5d10;
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2444,29 +2444,47 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
const curDimensions =
|
||||
deepGet(spec, `dataSchema.parser.parseSpec.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
|
||||
|
||||
const convertToMetricMenu = (
|
||||
<Menu>
|
||||
<MenuItem
|
||||
text="Convert to hyperUnique metric"
|
||||
onClick={() => {
|
||||
const convertToMetric = (type: string, prefix: string) => {
|
||||
const specWithoutDimension = deepDelete(
|
||||
spec,
|
||||
`dataSchema.parser.parseSpec.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
|
||||
);
|
||||
|
||||
const specWithMetric = deepSet(
|
||||
specWithoutDimension,
|
||||
`dataSchema.metricsSpec.[append]`,
|
||||
{
|
||||
name: `unique_${selectedDimensionSpec.name}`,
|
||||
type: 'hyperUnique',
|
||||
const specWithMetric = deepSet(specWithoutDimension, `dataSchema.metricsSpec.[append]`, {
|
||||
name: `${prefix}_${selectedDimensionSpec.name}`,
|
||||
type,
|
||||
fieldName: selectedDimensionSpec.name,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
this.updateSpec(specWithMetric);
|
||||
close();
|
||||
}}
|
||||
};
|
||||
|
||||
const convertToMetricMenu = (
|
||||
<Menu>
|
||||
<MenuItem
|
||||
text="Convert to longSum metric"
|
||||
onClick={() => convertToMetric('longSum', 'sum')}
|
||||
/>
|
||||
<MenuItem
|
||||
text="Convert to doubleSum metric"
|
||||
onClick={() => convertToMetric('doubleSum', 'sum')}
|
||||
/>
|
||||
<MenuItem
|
||||
text="Convert to thetaSketch metric"
|
||||
onClick={() => convertToMetric('thetaSketch', 'theta')}
|
||||
/>
|
||||
<MenuItem
|
||||
text="Convert to HLLSketchBuild metric"
|
||||
onClick={() => convertToMetric('HLLSketchBuild', 'hll')}
|
||||
/>
|
||||
<MenuItem
|
||||
text="Convert to quantilesDoublesSketch metric"
|
||||
onClick={() => convertToMetric('quantilesDoublesSketch', 'quantiles_doubles')}
|
||||
/>
|
||||
<MenuItem
|
||||
text="Convert to hyperUnique metric"
|
||||
onClick={() => convertToMetric('hyperUnique', 'unique')}
|
||||
/>
|
||||
</Menu>
|
||||
);
|
||||
|
|
@ -2483,7 +2501,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
<Popover content={convertToMetricMenu}>
|
||||
<Button
|
||||
icon={IconNames.EXCHANGE}
|
||||
text="Convert to metric..."
|
||||
text="Convert to metric"
|
||||
rightIcon={IconNames.CARET_DOWN}
|
||||
disabled={curDimensions.length <= 1}
|
||||
/>
|
||||
</Popover>
|
||||
|
|
@ -2602,7 +2621,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
{selectedMetricSpecIndex !== -1 && (
|
||||
<FormGroup>
|
||||
<Popover content={convertToDimensionMenu}>
|
||||
<Button icon={IconNames.EXCHANGE} text="Convert to dimension..." />
|
||||
<Button
|
||||
icon={IconNames.EXCHANGE}
|
||||
text="Convert to dimension"
|
||||
rightIcon={IconNames.CARET_DOWN}
|
||||
/>
|
||||
</Popover>
|
||||
</FormGroup>
|
||||
)}
|
||||
|
|
|
|||
Loading…
Reference in New Issue