Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: Support all possible metric types in the data loader (#8785) 

* Support all possible metric types in the data loader

* added more sketches
This commit is contained in:
Vadim Ogievetsky 2019-10-30 09:34:13 -07:00 committed by Fangjin Yang
parent a95e3d438e
commit 929a8b6337
5 changed files with 244 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap
View File

@ -59,6 +59,7 @@ exports[`retention dialog matches snapshot 1`] = `
<a
href="https://druid.apache.org/docs/0.16.0-incubating/operations/rule-configuration.html"
rel="noopener noreferrer"
target="_blank"
>
documentation

7
web-console/src/dialogs/retention-dialog/retention-dialog.tsx
View File

@ -21,7 +21,7 @@ import { IconNames } from '@blueprintjs/icons';
import axios from 'axios';
import React from 'react';
import { RuleEditor } from '../../components';
import { ExternalLink, RuleEditor } from '../../components';
import { QueryManager } from '../../utils';
import { DRUID_DOCS_VERSION } from '../../variables';
import { SnitchDialog } from '../snitch-dialog/snitch-dialog';
@ -183,12 +183,11 @@ export class RetentionDialog extends React.PureComponent<
<p>
Druid uses rules to determine what data should be retained in the cluster. The rules are
evaluated in order from top to bottom. For more information please refer to the{' '}
<a
<ExternalLink
href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/operations/rule-configuration.html`}
target="_blank"
>
documentation
</a>
</ExternalLink>
.
</p>
<FormGroup>{(currentRules || []).map(this.renderRule)}</FormGroup>

225
web-console/src/utils/ingestion-spec.tsx
View File

@ -668,6 +668,7 @@ const METRIC_SPEC_FORM_FIELDS: Field<MetricSpec>[] = [
{
name: 'name',
type: 'string',
info: <>The metric name as it will appear in Druid.</>,
},
{
name: 'type',
@ -694,37 +695,24 @@ const METRIC_SPEC_FORM_FIELDS: Field<MetricSpec>[] = [
group: 'last',
suggestions: ['longLast', 'doubleLast', 'floatLast'],
},
'cardinality',
'thetaSketch',
{
group: 'HLLSketch',
suggestions: ['HLLSketchBuild', 'HLLSketchMerge'],
},
'quantilesDoublesSketch',
'momentSketch',
'fixedBucketsHistogram',
'hyperUnique',
'filtered',
],
info: <>The aggregation function to apply.</>,
},
{
name: 'fieldName',
type: 'string',
defined: m => {
return [
'longSum',
'doubleSum',
'floatSum',
'longMin',
'doubleMin',
'floatMin',
'longMax',
'doubleMax',
'floatMax',
'longFirst',
'doubleFirst',
'floatFirst',
'stringFirst',
'longLast',
'doubleLast',
'floatLast',
'stringLast',
'cardinality',
'hyperUnique',
].includes(m.type);
},
defined: m => m.type !== 'filtered',
info: <>The column name for the aggregator to operate on.</>,
},
{
name: 'maxStringBytes',
@ -742,21 +730,196 @@ const METRIC_SPEC_FORM_FIELDS: Field<MetricSpec>[] = [
return ['stringFirst', 'stringLast'].includes(m.type);
},
},
// filtered
{
name: 'filter',
type: 'json',
defined: m => {
return m.type === 'filtered';
},
defined: m => m.type === 'filtered',
},
{
name: 'aggregator',
type: 'json',
defined: m => {
return m.type === 'filtered';
},
defined: m => m.type === 'filtered',
},
// thetaSketch
{
name: 'size',
type: 'number',
defined: m => m.type === 'thetaSketch',
defaultValue: 16384,
info: (
<>
<p>
Must be a power of 2. Internally, size refers to the maximum number of entries sketch
object will retain. Higher size means higher accuracy but more space to store sketches.
Note that after you index with a particular size, druid will persist sketch in segments
and you will use size greater or equal to that at query time.
</p>
<p>
See the{' '}
<ExternalLink href="https://datasketches.github.io/docs/Theta/ThetaSize.html">
DataSketches site
</ExternalLink>{' '}
for details.
</p>
<p>In general, We recommend just sticking to default size.</p>
</>
),
},
{
name: 'isInputThetaSketch',
type: 'boolean',
defined: m => m.type === 'thetaSketch',
defaultValue: false,
info: (
<>
This should only be used at indexing time if your input data contains theta sketch objects.
This would be the case if you use datasketches library outside of Druid, say with Pig/Hive,
to produce the data that you are ingesting into Druid
</>
),
},
// HLLSketchBuild & HLLSketchMerge
{
name: 'lgK',
type: 'number',
defined: m => m.type === 'HLLSketchBuild' || m.type === 'HLLSketchMerge',
defaultValue: 12,
info: (
<>
<p>
log2 of K that is the number of buckets in the sketch, parameter that controls the size
and the accuracy.
</p>
<p>Must be between 4 to 21 inclusively.</p>
</>
),
},
{
name: 'tgtHllType',
type: 'string',
defined: m => m.type === 'HLLSketchBuild' || m.type === 'HLLSketchMerge',
defaultValue: 'HLL_4',
suggestions: ['HLL_4', 'HLL_6', 'HLL_8'],
info: (
<>
The type of the target HLL sketch. Must be <Code>HLL_4</Code>, <Code>HLL_6</Code>, or{' '}
<Code>HLL_8</Code>.
</>
),
},
// quantilesDoublesSketch
{
name: 'k',
type: 'number',
defined: m => m.type === 'quantilesDoublesSketch',
defaultValue: 128,
info: (
<>
<p>
Parameter that determines the accuracy and size of the sketch. Higher k means higher
accuracy but more space to store sketches.
</p>
<p>
Must be a power of 2 from 2 to 32768. See the{' '}
<ExternalLink href="https://datasketches.github.io/docs/Quantiles/QuantilesAccuracy.html">
Quantiles Accuracy
</ExternalLink>{' '}
for details.
</p>
</>
),
},
// momentSketch
{
name: 'k',
type: 'number',
defined: m => m.type === 'momentSketch',
required: true,
info: (
<>
Parameter that determines the accuracy and size of the sketch. Higher k means higher
accuracy but more space to store sketches. Usable range is generally [3,15]
</>
),
},
{
name: 'compress',
type: 'boolean',
defined: m => m.type === 'momentSketch',
defaultValue: true,
info: (
<>
Flag for whether the aggregator compresses numeric values using arcsinh. Can improve
robustness to skewed and long-tailed distributions, but reduces accuracy slightly on more
uniform distributions.
</>
),
},
// fixedBucketsHistogram
{
name: 'lowerLimit',
type: 'number',
defined: m => m.type === 'fixedBucketsHistogram',
required: true,
info: <>Lower limit of the histogram.</>,
},
{
name: 'upperLimit',
type: 'number',
defined: m => m.type === 'fixedBucketsHistogram',
required: true,
info: <>Upper limit of the histogram.</>,
},
{
name: 'numBuckets',
type: 'number',
defined: m => m.type === 'fixedBucketsHistogram',
defaultValue: 10,
required: true,
info: (
<>
Number of buckets for the histogram. The range <Code>[lowerLimit, upperLimit]</Code> will be
divided into <Code>numBuckets</Code> intervals of equal size.
</>
),
},
{
name: 'outlierHandlingMode',
type: 'string',
defined: m => m.type === 'fixedBucketsHistogram',
required: true,
suggestions: ['ignore', 'overflow', 'clip'],
info: (
<>
<p>
Specifies how values outside of <Code>[lowerLimit, upperLimit]</Code> will be handled.
</p>
<p>
Supported modes are <Code>ignore</Code>, <Code>overflow</Code>, and <Code>clip</Code>. See
<ExternalLink
href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-core/approximate-histograms.html#outlier-handling-modes`}
>
outlier handling modes
</ExternalLink>{' '}
for more details.
</p>
</>
),
},
// hyperUnique
{
name: 'isInputHyperUnique',
type: 'boolean',
defined: m => m.type === 'hyperUnique',
defaultValue: false,
info: (
<>
This can be set to true to index precomputed HLL (Base64 encoded output from druid-hll is
expected).
</>
),
},
// ToDo: fill in approximates
];
export function getMetricSpecFormFields() {

4
web-console/src/views/load-data-view/load-data-view.scss
View File

@ -88,7 +88,7 @@
bottom: 0;
left: 0;
content: '';
border: 2px solid #008adc;
border: 2px solid #48aff0;
border-radius: 2px;
}
@ -195,7 +195,7 @@
left: 0;
right: 0;
content: '';
border: 2px solid #008adc;
border: 2px solid #ff5d10;
border-radius: 2px;
}

65
web-console/src/views/load-data-view/load-data-view.tsx
View File

@ -2444,29 +2444,47 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const curDimensions =
deepGet(spec, `dataSchema.parser.parseSpec.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
const convertToMetric = (type: string, prefix: string) => {
const specWithoutDimension = deepDelete(
spec,
`dataSchema.parser.parseSpec.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
);
const specWithMetric = deepSet(specWithoutDimension, `dataSchema.metricsSpec.[append]`, {
name: `${prefix}_${selectedDimensionSpec.name}`,
type,
fieldName: selectedDimensionSpec.name,
});
this.updateSpec(specWithMetric);
close();
};
const convertToMetricMenu = (
<Menu>
<MenuItem
text="Convert to longSum metric"
onClick={() => convertToMetric('longSum', 'sum')}
/>
<MenuItem
text="Convert to doubleSum metric"
onClick={() => convertToMetric('doubleSum', 'sum')}
/>
<MenuItem
text="Convert to thetaSketch metric"
onClick={() => convertToMetric('thetaSketch', 'theta')}
/>
<MenuItem
text="Convert to HLLSketchBuild metric"
onClick={() => convertToMetric('HLLSketchBuild', 'hll')}
/>
<MenuItem
text="Convert to quantilesDoublesSketch metric"
onClick={() => convertToMetric('quantilesDoublesSketch', 'quantiles_doubles')}
/>
<MenuItem
text="Convert to hyperUnique metric"
onClick={() => {
const specWithoutDimension = deepDelete(
spec,
`dataSchema.parser.parseSpec.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
);
const specWithMetric = deepSet(
specWithoutDimension,
`dataSchema.metricsSpec.[append]`,
{
name: `unique_${selectedDimensionSpec.name}`,
type: 'hyperUnique',
fieldName: selectedDimensionSpec.name,
},
);
this.updateSpec(specWithMetric);
close();
}}
onClick={() => convertToMetric('hyperUnique', 'unique')}
/>
</Menu>
);
@ -2483,7 +2501,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
<Popover content={convertToMetricMenu}>
<Button
icon={IconNames.EXCHANGE}
text="Convert to metric..."
text="Convert to metric"
rightIcon={IconNames.CARET_DOWN}
disabled={curDimensions.length <= 1}
/>
</Popover>
@ -2602,7 +2621,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
{selectedMetricSpecIndex !== -1 && (
<FormGroup>
<Popover content={convertToDimensionMenu}>
<Button icon={IconNames.EXCHANGE} text="Convert to dimension..." />
<Button
icon={IconNames.EXCHANGE}
text="Convert to dimension"
rightIcon={IconNames.CARET_DOWN}
/>
</Popover>
</FormGroup>
)}