mirror of https://github.com/apache/druid.git
Web console: Adding a shard detail column to the segments view (#12212)
* shard spec details * improve pattern match * refactor spec cleanup * better format detection * update JSONbig * add multiline option to autoform
This commit is contained in:
Vadim Ogievetsky
GitHub
parent
801d9e7f1b
commit
bc408bacc8
|
|
@ -5813,7 +5813,7 @@ license_category: binary
|
|||
module: web-console
|
||||
license_name: MIT License
|
||||
copyright: Vadim Ogievetsky, Andrey Sidorov
|
||||
version: 1.1.0
|
||||
version: 1.2.0
|
||||
license_file_path: licenses/bin/json-bigint-native.MIT
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -14771,9 +14771,9 @@
|
|||
"integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA=="
|
||||
},
|
||||
"json-bigint-native": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/json-bigint-native/-/json-bigint-native-1.1.0.tgz",
|
||||
"integrity": "sha512-PPL9AlDP0ift5v8siEsR7oQsamOAIOLjn14GRaijZRUWDXsJC5rHXNlmtLPkPjK0k2i5yHK30VPqiFTZHolXaA=="
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/json-bigint-native/-/json-bigint-native-1.2.0.tgz",
|
||||
"integrity": "sha512-qC9EtJsyULhbwC2KEYoR8sRsC+PH7VwwPdxU6+CZTZxMtM23zlxCfhIa+6Sn74FQ4VqDqWUaHaBeU0bMUTU9jQ=="
|
||||
},
|
||||
"json-parse-better-errors": {
|
||||
"version": "1.0.2",
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@
|
|||
"fontsource-open-sans": "^3.0.9",
|
||||
"has-own-prop": "^2.0.0",
|
||||
"hjson": "^3.2.1",
|
||||
"json-bigint-native": "^1.1.0",
|
||||
"json-bigint-native": "^1.2.0",
|
||||
"lodash.debounce": "^4.0.8",
|
||||
"lodash.escape": "^4.0.1",
|
||||
"memoize-one": "^5.1.1",
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ exports[`AutoForm matches snapshot 1`] = `
|
|||
>
|
||||
<Memo(SuggestibleInput)
|
||||
disabled={false}
|
||||
multiline={false}
|
||||
onBlur={[Function]}
|
||||
onValueChange={[Function]}
|
||||
placeholder=""
|
||||
|
|
@ -57,6 +58,20 @@ exports[`AutoForm matches snapshot 1`] = `
|
|||
>
|
||||
<Memo(SuggestibleInput)
|
||||
disabled={false}
|
||||
multiline={false}
|
||||
onBlur={[Function]}
|
||||
onValueChange={[Function]}
|
||||
placeholder=""
|
||||
value="Hello World"
|
||||
/>
|
||||
</Memo(FormGroupWithInfo)>
|
||||
<Memo(FormGroupWithInfo)
|
||||
key="testStringWithMultiline"
|
||||
label="Test string with multiline"
|
||||
>
|
||||
<Memo(SuggestibleInput)
|
||||
disabled={false}
|
||||
multiline={true}
|
||||
onBlur={[Function]}
|
||||
onValueChange={[Function]}
|
||||
placeholder=""
|
||||
|
|
@ -148,6 +163,7 @@ exports[`AutoForm matches snapshot 1`] = `
|
|||
<Memo(SuggestibleInput)
|
||||
disabled={false}
|
||||
intent="primary"
|
||||
multiline={false}
|
||||
onBlur={[Function]}
|
||||
onValueChange={[Function]}
|
||||
placeholder=""
|
||||
|
|
|
|||
|
|
@ -32,6 +32,12 @@ describe('AutoForm', () => {
|
|||
{ name: 'testSizeBytes', type: 'size-bytes' },
|
||||
{ name: 'testString', type: 'string' },
|
||||
{ name: 'testStringWithDefault', type: 'string', defaultValue: 'Hello World' },
|
||||
{
|
||||
name: 'testStringWithMultiline',
|
||||
type: 'string',
|
||||
multiline: true,
|
||||
defaultValue: 'Hello World',
|
||||
},
|
||||
{ name: 'testBoolean', type: 'boolean' },
|
||||
{ name: 'testBooleanWithDefault', type: 'boolean', defaultValue: false },
|
||||
{ name: 'testStringArray', type: 'string-array' },
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ export interface Field<M> {
|
|||
disabled?: Functor<M, boolean>;
|
||||
defined?: Functor<M, boolean>;
|
||||
required?: Functor<M, boolean>;
|
||||
multiline?: Functor<M, boolean>;
|
||||
hide?: Functor<M, boolean>;
|
||||
hideInMore?: Functor<M, boolean>;
|
||||
valueAdjustment?: (value: any) => any;
|
||||
|
|
@ -303,6 +304,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
|
|||
large={large}
|
||||
disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
|
||||
intent={required && modelValue == null ? AutoForm.REQUIRED_INTENT : undefined}
|
||||
multiline={AutoForm.evaluateFunctor(field.multiline, model, false)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,3 +31,15 @@ exports[`FormattedInput matches snapshot with escaped value 1`] = `
|
|||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
exports[`FormattedInput matches works with multiline 1`] = `
|
||||
<div
|
||||
class="formatted-input"
|
||||
>
|
||||
<textarea
|
||||
class="bp3-input"
|
||||
>
|
||||
Here are some chars \\t\\r\\n lol
|
||||
</textarea>
|
||||
</div>
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -26,4 +26,9 @@
|
|||
top: 0;
|
||||
bottom: 0;
|
||||
}
|
||||
|
||||
textarea {
|
||||
width: 100%;
|
||||
resize: vertical;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,4 +45,18 @@ describe('FormattedInput', () => {
|
|||
const { container } = render(suggestibleInput);
|
||||
expect(container.firstChild).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('matches works with multiline', () => {
|
||||
const suggestibleInput = (
|
||||
<FormattedInput
|
||||
value={`Here are some chars \t\r\n lol`}
|
||||
onValueChange={() => {}}
|
||||
formatter={JSON_STRING_FORMATTER}
|
||||
multiline
|
||||
/>
|
||||
);
|
||||
|
||||
const { container } = render(suggestibleInput);
|
||||
expect(container.firstChild).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { InputGroup, InputGroupProps2, Intent } from '@blueprintjs/core';
|
||||
import { InputGroup, InputGroupProps2, Intent, TextArea } from '@blueprintjs/core';
|
||||
import { Tooltip2 } from '@blueprintjs/popover2';
|
||||
import classNames from 'classnames';
|
||||
import React, { useState } from 'react';
|
||||
|
|
@ -30,6 +30,7 @@ export interface FormattedInputProps extends InputGroupProps2 {
|
|||
onValueChange: (newValue: undefined | string) => void;
|
||||
sanitizer?: (rawValue: string) => string;
|
||||
issueWithValue?: (value: any) => string | undefined;
|
||||
multiline?: boolean;
|
||||
}
|
||||
|
||||
export const FormattedInput = React.memo(function FormattedInput(props: FormattedInputProps) {
|
||||
|
|
@ -44,6 +45,8 @@ export const FormattedInput = React.memo(function FormattedInput(props: Formatte
|
|||
onFocus,
|
||||
onBlur,
|
||||
intent,
|
||||
placeholder,
|
||||
multiline,
|
||||
...rest
|
||||
} = props;
|
||||
|
||||
|
|
@ -53,20 +56,17 @@ export const FormattedInput = React.memo(function FormattedInput(props: Formatte
|
|||
const issue: string | undefined = issueWithValue?.(value);
|
||||
const showIssue = Boolean(!isFocused && issue);
|
||||
|
||||
return (
|
||||
<div className={classNames('formatted-input', className)}>
|
||||
<InputGroup
|
||||
value={
|
||||
const myValue =
|
||||
typeof intermediateValue !== 'undefined'
|
||||
? intermediateValue
|
||||
: typeof value !== 'undefined'
|
||||
? formatter.stringify(value)
|
||||
: undefined
|
||||
}
|
||||
defaultValue={
|
||||
typeof defaultValue !== 'undefined' ? formatter.stringify(defaultValue) : undefined
|
||||
}
|
||||
onChange={e => {
|
||||
: undefined;
|
||||
|
||||
const myDefaultValue =
|
||||
typeof defaultValue !== 'undefined' ? formatter.stringify(defaultValue) : undefined;
|
||||
|
||||
const myOnChange = (e: any) => {
|
||||
let rawValue = e.target.value;
|
||||
if (sanitizer) rawValue = sanitizer(rawValue);
|
||||
setIntermediateValue(rawValue);
|
||||
|
|
@ -78,19 +78,45 @@ export const FormattedInput = React.memo(function FormattedInput(props: Formatte
|
|||
return;
|
||||
}
|
||||
onValueChange(parsedValue);
|
||||
}}
|
||||
onFocus={e => {
|
||||
};
|
||||
|
||||
const myOnFocus = (e: any) => {
|
||||
setIsFocused(true);
|
||||
onFocus?.(e);
|
||||
}}
|
||||
onBlur={e => {
|
||||
};
|
||||
|
||||
const myOnBlur = (e: any) => {
|
||||
setIntermediateValue(undefined);
|
||||
setIsFocused(false);
|
||||
onBlur?.(e);
|
||||
}}
|
||||
intent={showIssue ? Intent.DANGER : intent}
|
||||
};
|
||||
|
||||
const myIntent = showIssue ? Intent.DANGER : intent;
|
||||
|
||||
return (
|
||||
<div className={classNames('formatted-input', className)}>
|
||||
{multiline ? (
|
||||
<TextArea
|
||||
value={myValue}
|
||||
defaultValue={myDefaultValue}
|
||||
onChange={myOnChange}
|
||||
onFocus={myOnFocus}
|
||||
onBlur={myOnBlur}
|
||||
intent={myIntent}
|
||||
placeholder={placeholder}
|
||||
/>
|
||||
) : (
|
||||
<InputGroup
|
||||
value={myValue}
|
||||
defaultValue={myDefaultValue}
|
||||
onChange={myOnChange}
|
||||
onFocus={myOnFocus}
|
||||
onBlur={myOnBlur}
|
||||
intent={myIntent}
|
||||
placeholder={placeholder}
|
||||
{...rest}
|
||||
/>
|
||||
)}
|
||||
{showIssue && (
|
||||
<Tooltip2
|
||||
isOpen
|
||||
|
|
|
|||
|
|
@ -330,6 +330,185 @@ describe('ingestion-spec', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('upgrades / downgrades back compat supervisor spec', () => {
|
||||
const backCompatSupervisorSpec = {
|
||||
type: 'kafka',
|
||||
spec: {
|
||||
dataSchema: {
|
||||
dataSource: 'metrics-kafka',
|
||||
parser: {
|
||||
type: 'string',
|
||||
parseSpec: {
|
||||
format: 'json',
|
||||
timestampSpec: {
|
||||
column: 'timestamp',
|
||||
format: 'auto',
|
||||
},
|
||||
dimensionsSpec: {
|
||||
dimensions: [],
|
||||
dimensionExclusions: ['timestamp', 'value'],
|
||||
},
|
||||
},
|
||||
},
|
||||
metricsSpec: [
|
||||
{
|
||||
name: 'count',
|
||||
type: 'count',
|
||||
},
|
||||
{
|
||||
name: 'value_sum',
|
||||
fieldName: 'value',
|
||||
type: 'doubleSum',
|
||||
},
|
||||
{
|
||||
name: 'value_min',
|
||||
fieldName: 'value',
|
||||
type: 'doubleMin',
|
||||
},
|
||||
{
|
||||
name: 'value_max',
|
||||
fieldName: 'value',
|
||||
type: 'doubleMax',
|
||||
},
|
||||
],
|
||||
granularitySpec: {
|
||||
type: 'uniform',
|
||||
segmentGranularity: 'HOUR',
|
||||
queryGranularity: 'NONE',
|
||||
},
|
||||
},
|
||||
tuningConfig: {
|
||||
type: 'kafka',
|
||||
maxRowsPerSegment: 5000000,
|
||||
},
|
||||
ioConfig: {
|
||||
topic: 'metrics',
|
||||
consumerProperties: {
|
||||
'bootstrap.servers': 'localhost:9092',
|
||||
},
|
||||
taskCount: 1,
|
||||
replicas: 1,
|
||||
taskDuration: 'PT1H',
|
||||
},
|
||||
},
|
||||
dataSchema: {
|
||||
dataSource: 'metrics-kafka',
|
||||
parser: {
|
||||
type: 'string',
|
||||
parseSpec: {
|
||||
format: 'json',
|
||||
timestampSpec: {
|
||||
column: 'timestamp',
|
||||
format: 'auto',
|
||||
},
|
||||
dimensionsSpec: {
|
||||
dimensions: [],
|
||||
dimensionExclusions: ['timestamp', 'value'],
|
||||
},
|
||||
},
|
||||
},
|
||||
metricsSpec: [
|
||||
{
|
||||
name: 'count',
|
||||
type: 'count',
|
||||
},
|
||||
{
|
||||
name: 'value_sum',
|
||||
fieldName: 'value',
|
||||
type: 'doubleSum',
|
||||
},
|
||||
{
|
||||
name: 'value_min',
|
||||
fieldName: 'value',
|
||||
type: 'doubleMin',
|
||||
},
|
||||
{
|
||||
name: 'value_max',
|
||||
fieldName: 'value',
|
||||
type: 'doubleMax',
|
||||
},
|
||||
],
|
||||
granularitySpec: {
|
||||
type: 'uniform',
|
||||
segmentGranularity: 'HOUR',
|
||||
queryGranularity: 'NONE',
|
||||
},
|
||||
},
|
||||
tuningConfig: {
|
||||
type: 'kafka',
|
||||
maxRowsPerSegment: 5000000,
|
||||
},
|
||||
ioConfig: {
|
||||
topic: 'metrics',
|
||||
consumerProperties: {
|
||||
'bootstrap.servers': 'localhost:9092',
|
||||
},
|
||||
taskCount: 1,
|
||||
replicas: 1,
|
||||
taskDuration: 'PT1H',
|
||||
},
|
||||
};
|
||||
|
||||
expect(cleanSpec(upgradeSpec(backCompatSupervisorSpec))).toEqual({
|
||||
spec: {
|
||||
dataSchema: {
|
||||
dataSource: 'metrics-kafka',
|
||||
dimensionsSpec: {
|
||||
dimensionExclusions: ['timestamp', 'value'],
|
||||
dimensions: [],
|
||||
},
|
||||
granularitySpec: {
|
||||
queryGranularity: 'NONE',
|
||||
segmentGranularity: 'HOUR',
|
||||
type: 'uniform',
|
||||
},
|
||||
metricsSpec: [
|
||||
{
|
||||
name: 'count',
|
||||
type: 'count',
|
||||
},
|
||||
{
|
||||
fieldName: 'value',
|
||||
name: 'value_sum',
|
||||
type: 'doubleSum',
|
||||
},
|
||||
{
|
||||
fieldName: 'value',
|
||||
name: 'value_min',
|
||||
type: 'doubleMin',
|
||||
},
|
||||
{
|
||||
fieldName: 'value',
|
||||
name: 'value_max',
|
||||
type: 'doubleMax',
|
||||
},
|
||||
],
|
||||
timestampSpec: {
|
||||
column: 'timestamp',
|
||||
format: 'auto',
|
||||
},
|
||||
},
|
||||
ioConfig: {
|
||||
consumerProperties: {
|
||||
'bootstrap.servers': 'localhost:9092',
|
||||
},
|
||||
inputFormat: {
|
||||
type: 'json',
|
||||
},
|
||||
replicas: 1,
|
||||
taskCount: 1,
|
||||
taskDuration: 'PT1H',
|
||||
topic: 'metrics',
|
||||
},
|
||||
tuningConfig: {
|
||||
maxRowsPerSegment: 5000000,
|
||||
type: 'kafka',
|
||||
},
|
||||
},
|
||||
type: 'kafka',
|
||||
});
|
||||
});
|
||||
|
||||
it('cleanSpec', () => {
|
||||
expect(
|
||||
cleanSpec({
|
||||
|
|
@ -451,6 +630,10 @@ describe('spec utils', () => {
|
|||
expect(guessColumnTypeFromInput(['a', ['b'], 'c'], false)).toEqual('string');
|
||||
expect(guessColumnTypeFromInput([1, [2], 3], false)).toEqual('string');
|
||||
});
|
||||
|
||||
it('works for strange input (object with no prototype)', () => {
|
||||
expect(guessColumnTypeFromInput([1, Object.create(null), 3], false)).toEqual('string');
|
||||
});
|
||||
});
|
||||
|
||||
describe('guessColumnTypeFromHeaderAndRows', () => {
|
||||
|
|
|
|||
|
|
@ -291,6 +291,23 @@ export function isDruidSource(spec: Partial<IngestionSpec>): boolean {
|
|||
return deepGet(spec, 'spec.ioConfig.inputSource.type') === 'druid';
|
||||
}
|
||||
|
||||
// ---------------------------------
|
||||
// Spec cleanup and normalization
|
||||
|
||||
/**
|
||||
* Make sure that the ioConfig, dataSchema, e.t.c. are nested inside of spec and not just hanging out at the top level
|
||||
* @param spec
|
||||
*/
|
||||
function nestSpecIfNeeded(spec: any): Partial<IngestionSpec> {
|
||||
if (spec?.type && typeof spec.spec !== 'object' && (spec.ioConfig || spec.dataSchema)) {
|
||||
return {
|
||||
type: spec.type,
|
||||
spec: deepDelete(spec, 'type'),
|
||||
};
|
||||
}
|
||||
return spec;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure that the types are set in the root, ioConfig, and tuningConfig
|
||||
* @param spec
|
||||
|
|
@ -301,10 +318,7 @@ export function normalizeSpec(spec: Partial<IngestionSpec>): IngestionSpec {
|
|||
spec = {};
|
||||
}
|
||||
|
||||
// Make sure that if we actually get a task payload we extract the spec
|
||||
if (typeof spec.spec !== 'object' && typeof (spec as any).ioConfig === 'object') {
|
||||
spec = { spec: spec as any };
|
||||
}
|
||||
spec = nestSpecIfNeeded(spec);
|
||||
|
||||
const specType =
|
||||
deepGet(spec, 'type') ||
|
||||
|
|
@ -333,6 +347,47 @@ export function cleanSpec(
|
|||
) as IngestionSpec;
|
||||
}
|
||||
|
||||
export function upgradeSpec(spec: any): Partial<IngestionSpec> {
|
||||
spec = nestSpecIfNeeded(spec);
|
||||
|
||||
// Upgrade firehose if exists
|
||||
if (deepGet(spec, 'spec.ioConfig.firehose')) {
|
||||
switch (deepGet(spec, 'spec.ioConfig.firehose.type')) {
|
||||
case 'static-s3':
|
||||
deepSet(spec, 'spec.ioConfig.firehose.type', 's3');
|
||||
break;
|
||||
|
||||
case 'static-google-blobstore':
|
||||
deepSet(spec, 'spec.ioConfig.firehose.type', 'google');
|
||||
deepMove(spec, 'spec.ioConfig.firehose.blobs', 'spec.ioConfig.firehose.objects');
|
||||
break;
|
||||
}
|
||||
|
||||
spec = deepMove(spec, 'spec.ioConfig.firehose', 'spec.ioConfig.inputSource');
|
||||
}
|
||||
|
||||
// Decompose parser if exists
|
||||
if (deepGet(spec, 'spec.dataSchema.parser')) {
|
||||
spec = deepMove(
|
||||
spec,
|
||||
'spec.dataSchema.parser.parseSpec.timestampSpec',
|
||||
'spec.dataSchema.timestampSpec',
|
||||
);
|
||||
spec = deepMove(
|
||||
spec,
|
||||
'spec.dataSchema.parser.parseSpec.dimensionsSpec',
|
||||
'spec.dataSchema.dimensionsSpec',
|
||||
);
|
||||
spec = deepMove(spec, 'spec.dataSchema.parser.parseSpec', 'spec.ioConfig.inputFormat');
|
||||
spec = deepDelete(spec, 'spec.dataSchema.parser');
|
||||
spec = deepMove(spec, 'spec.ioConfig.inputFormat.format', 'spec.ioConfig.inputFormat.type');
|
||||
}
|
||||
|
||||
return spec;
|
||||
}
|
||||
|
||||
// ------------------------------------
|
||||
|
||||
export interface GranularitySpec {
|
||||
type?: string;
|
||||
queryGranularity?: string;
|
||||
|
|
@ -2192,11 +2247,12 @@ export function guessColumnTypeFromInput(
|
|||
if (definedValues.some(v => Array.isArray(v))) return 'string';
|
||||
|
||||
if (
|
||||
definedValues.every(
|
||||
v =>
|
||||
!isNaN(v) &&
|
||||
(typeof v === 'number' || (guessNumericStringsAsNumbers && typeof v === 'string')),
|
||||
)
|
||||
definedValues.every(v => {
|
||||
return (
|
||||
(typeof v === 'number' || (guessNumericStringsAsNumbers && typeof v === 'string')) &&
|
||||
!isNaN(Number(v))
|
||||
);
|
||||
})
|
||||
) {
|
||||
return definedValues.every(v => v % 1 === 0) ? 'long' : 'double';
|
||||
} else {
|
||||
|
|
@ -2215,6 +2271,10 @@ export function guessColumnTypeFromHeaderAndRows(
|
|||
);
|
||||
}
|
||||
|
||||
export function inputFormatOutputsNumericStrings(inputFormat: InputFormat | undefined): boolean {
|
||||
return oneOf(inputFormat?.type, 'csv', 'tsv', 'regex');
|
||||
}
|
||||
|
||||
function getTypeHintsFromSpec(spec: Partial<IngestionSpec>): Record<string, string> {
|
||||
const typeHints: Record<string, string> = {};
|
||||
const currentDimensions = deepGet(spec, 'spec.dataSchema.dimensionsSpec.dimensions') || [];
|
||||
|
|
@ -2242,7 +2302,9 @@ export function updateSchemaWithSample(
|
|||
forcePartitionInitialization = false,
|
||||
): Partial<IngestionSpec> {
|
||||
const typeHints = getTypeHintsFromSpec(spec);
|
||||
const guessNumericStringsAsNumbers = deepGet(spec, 'spec.ioConfig.inputFormat.type') !== 'json';
|
||||
const guessNumericStringsAsNumbers = inputFormatOutputsNumericStrings(
|
||||
deepGet(spec, 'spec.ioConfig.inputFormat'),
|
||||
);
|
||||
|
||||
let newSpec = spec;
|
||||
|
||||
|
|
@ -2292,52 +2354,6 @@ export function updateSchemaWithSample(
|
|||
return newSpec;
|
||||
}
|
||||
|
||||
// ------------------------
|
||||
|
||||
export function upgradeSpec(spec: any): Partial<IngestionSpec> {
|
||||
if (deepGet(spec, 'type') && deepGet(spec, 'dataSchema')) {
|
||||
spec = {
|
||||
type: spec.type,
|
||||
spec: deepDelete(spec, 'type'),
|
||||
};
|
||||
}
|
||||
|
||||
// Upgrade firehose if exists
|
||||
if (deepGet(spec, 'spec.ioConfig.firehose')) {
|
||||
switch (deepGet(spec, 'spec.ioConfig.firehose.type')) {
|
||||
case 'static-s3':
|
||||
deepSet(spec, 'spec.ioConfig.firehose.type', 's3');
|
||||
break;
|
||||
|
||||
case 'static-google-blobstore':
|
||||
deepSet(spec, 'spec.ioConfig.firehose.type', 'google');
|
||||
deepMove(spec, 'spec.ioConfig.firehose.blobs', 'spec.ioConfig.firehose.objects');
|
||||
break;
|
||||
}
|
||||
|
||||
spec = deepMove(spec, 'spec.ioConfig.firehose', 'spec.ioConfig.inputSource');
|
||||
}
|
||||
|
||||
// Decompose parser if exists
|
||||
if (deepGet(spec, 'spec.dataSchema.parser')) {
|
||||
spec = deepMove(
|
||||
spec,
|
||||
'spec.dataSchema.parser.parseSpec.timestampSpec',
|
||||
'spec.dataSchema.timestampSpec',
|
||||
);
|
||||
spec = deepMove(
|
||||
spec,
|
||||
'spec.dataSchema.parser.parseSpec.dimensionsSpec',
|
||||
'spec.dataSchema.dimensionsSpec',
|
||||
);
|
||||
spec = deepMove(spec, 'spec.dataSchema.parser.parseSpec', 'spec.ioConfig.inputFormat');
|
||||
spec = deepDelete(spec, 'spec.dataSchema.parser');
|
||||
spec = deepMove(spec, 'spec.ioConfig.inputFormat.format', 'spec.ioConfig.inputFormat.type');
|
||||
}
|
||||
|
||||
return spec;
|
||||
}
|
||||
|
||||
export function adjustId(id: string): string {
|
||||
return id
|
||||
.replace(/\//g, '') // Can not have /
|
||||
|
|
|
|||
|
|
@ -250,7 +250,7 @@ export async function sampleForConnect(
|
|||
if (!reingestMode) {
|
||||
ioConfig = deepSet(ioConfig, 'inputFormat', {
|
||||
type: 'regex',
|
||||
pattern: '(.*)',
|
||||
pattern: '([\\s\\S]*)', // Match the entire line, every single character
|
||||
listDelimiter: '56616469-6de2-9da4-efb8-8f416e6e6965', // Just a UUID to disable the list delimiter, let's hope we do not see this UUID in the data
|
||||
columns: ['raw'],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ exports[`SegmentsView matches snapshot 1`] = `
|
|||
"Version",
|
||||
"Time span",
|
||||
"Partitioning",
|
||||
"Shard detail",
|
||||
"Partition",
|
||||
"Size",
|
||||
"Num rows",
|
||||
|
|
@ -74,6 +75,7 @@ exports[`SegmentsView matches snapshot 1`] = `
|
|||
Array [
|
||||
"Time span",
|
||||
"Partitioning",
|
||||
"Shard detail",
|
||||
]
|
||||
}
|
||||
/>
|
||||
|
|
@ -206,6 +208,15 @@ exports[`SegmentsView matches snapshot 1`] = `
|
|||
"sortable": true,
|
||||
"width": 100,
|
||||
},
|
||||
Object {
|
||||
"Cell": [Function],
|
||||
"Header": "Shard detail",
|
||||
"accessor": "shard_spec",
|
||||
"filterable": false,
|
||||
"show": false,
|
||||
"sortable": false,
|
||||
"width": 400,
|
||||
},
|
||||
Object {
|
||||
"Header": "Partition",
|
||||
"accessor": "partition_num",
|
||||
|
|
|
|||
|
|
@ -31,6 +31,17 @@
|
|||
.-totalPages {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.range-detail {
|
||||
cursor: default;
|
||||
|
||||
.range-label {
|
||||
display: inline-block;
|
||||
width: 35px;
|
||||
text-align: right;
|
||||
margin-right: 5px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
&.show-segment-timeline {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
import { Button, ButtonGroup, Intent, Label, MenuItem, Switch } from '@blueprintjs/core';
|
||||
import { IconNames } from '@blueprintjs/icons';
|
||||
import classNames from 'classnames';
|
||||
import { SqlExpression, SqlRef } from 'druid-query-toolkit';
|
||||
import { SqlExpression, SqlLiteral, SqlRef } from 'druid-query-toolkit';
|
||||
import React from 'react';
|
||||
import ReactTable, { Filter } from 'react-table';
|
||||
|
||||
|
|
@ -75,6 +75,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
|
|||
'Version',
|
||||
'Time span',
|
||||
'Partitioning',
|
||||
'Shard detail',
|
||||
'Partition',
|
||||
'Size',
|
||||
'Num rows',
|
||||
|
|
@ -103,6 +104,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
|
|||
'End',
|
||||
'Version',
|
||||
'Partitioning',
|
||||
'Shard detail',
|
||||
'Partition',
|
||||
'Size',
|
||||
'Num rows',
|
||||
|
|
@ -115,6 +117,10 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
|
|||
],
|
||||
};
|
||||
|
||||
function formatRangeDimensionValue(dimension: any, value: any): string {
|
||||
return `${SqlRef.column(String(dimension))}=${SqlLiteral.create(String(value))}`;
|
||||
}
|
||||
|
||||
export interface SegmentsViewProps {
|
||||
goToQuery: (initSql: string) => void;
|
||||
datasource: string | undefined;
|
||||
|
|
@ -149,8 +155,9 @@ interface SegmentQueryResultRow {
|
|||
version: string;
|
||||
time_span: string;
|
||||
partitioning: string;
|
||||
size: number;
|
||||
shard_spec: string;
|
||||
partition_num: number;
|
||||
size: number;
|
||||
num_rows: NumberLike;
|
||||
avg_row_size: NumberLike;
|
||||
num_replicas: number;
|
||||
|
|
@ -202,6 +209,7 @@ END AS "time_span"`,
|
|||
WHEN "shard_spec" LIKE '%"type":"numbered_overwrite"%' THEN 'numbered_overwrite'
|
||||
ELSE '-'
|
||||
END AS "partitioning"`,
|
||||
visibleColumns.shown('Shard detail') && `"shard_spec"`,
|
||||
visibleColumns.shown('Partition') && `"partition_num"`,
|
||||
visibleColumns.shown('Size') && `"size"`,
|
||||
visibleColumns.shown('Num rows') && `"num_rows"`,
|
||||
|
|
@ -258,7 +266,7 @@ END AS "partitioning"`,
|
|||
segmentFilter,
|
||||
visibleColumns: new LocalStorageBackedVisibility(
|
||||
LocalStorageKeys.SEGMENT_TABLE_COLUMN_SELECTION,
|
||||
['Time span', 'Partitioning'],
|
||||
['Time span', 'Partitioning', 'Shard detail'],
|
||||
),
|
||||
groupByInterval: false,
|
||||
showSegmentTimeline: false,
|
||||
|
|
@ -394,6 +402,7 @@ END AS "partitioning"`,
|
|||
version: segment.version,
|
||||
time_span: SegmentsView.computeTimeSpan(start, end),
|
||||
partitioning: deepGet(segment, 'shardSpec.type') || '-',
|
||||
shard_spec: deepGet(segment, 'shardSpec'),
|
||||
partition_num: deepGet(segment, 'shardSpec.partitionNum') || 0,
|
||||
size: segment.size,
|
||||
num_rows: -1,
|
||||
|
|
@ -596,6 +605,67 @@ END AS "partitioning"`,
|
|||
filterable: allowGeneralFilter,
|
||||
Cell: renderFilterableCell('partitioning'),
|
||||
},
|
||||
{
|
||||
Header: 'Shard detail',
|
||||
show: visibleColumns.shown('Shard detail'),
|
||||
accessor: 'shard_spec',
|
||||
width: 400,
|
||||
sortable: false,
|
||||
filterable: false,
|
||||
Cell: ({ value }) => {
|
||||
let v: any;
|
||||
try {
|
||||
v = JSON.parse(value);
|
||||
} catch {
|
||||
return '-';
|
||||
}
|
||||
|
||||
switch (v?.type) {
|
||||
case 'range': {
|
||||
const dimensions = v.dimensions || [];
|
||||
const formatEdge = (values: string[]) =>
|
||||
values.map((x, i) => formatRangeDimensionValue(dimensions[i], x)).join('; ');
|
||||
|
||||
return (
|
||||
<div className="range-detail">
|
||||
<span className="range-label">Start:</span>
|
||||
{Array.isArray(v.start) ? formatEdge(v.start) : '-∞'}
|
||||
<br />
|
||||
<span className="range-label">End:</span>
|
||||
{Array.isArray(v.end) ? formatEdge(v.end) : '∞'}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
case 'single': {
|
||||
return (
|
||||
<div className="range-detail">
|
||||
<span className="range-label">Start:</span>
|
||||
{v.start != null ? formatRangeDimensionValue(v.dimension, v.start) : '-∞'}
|
||||
<br />
|
||||
<span className="range-label">End:</span>
|
||||
{v.end != null ? formatRangeDimensionValue(v.dimension, v.end) : '∞'}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
case 'hashed': {
|
||||
const { partitionDimensions } = v;
|
||||
if (!Array.isArray(partitionDimensions)) return value;
|
||||
return `Partition dimensions: ${
|
||||
partitionDimensions.length ? partitionDimensions.join('; ') : 'all'
|
||||
}`;
|
||||
}
|
||||
|
||||
case 'numbered':
|
||||
case 'none':
|
||||
return '-';
|
||||
|
||||
default:
|
||||
return typeof value === 'string' ? value : '-';
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
Header: 'Partition',
|
||||
show: visibleColumns.shown('Partition'),
|
||||
|
|
|
|||
Loading…
Reference in New Issue