Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: fix data loader schema table column ordering bug and other polish (#10588) 

* remove unused fields

* keep tables live

* advanced

* fix schema view

* better indication

* tests pass

* Show more instead of show advanced

* fix tests

* extract dynamic configs

* update snapshots

* fix issues

* update snapshot

* reword without >
This commit is contained in:
Vadim Ogievetsky 2020-11-17 13:25:03 -08:00 committed by GitHub
parent 3447934a75
commit 9964dd4cb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 1703 additions and 1012 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
web-console/e2e-tests/component/load-data/config/partition.ts
View File

@ -26,10 +26,10 @@ import { getLabeledInput, selectSuggestibleInput, setLabeledInput } from '../../
* Possible values for partition step segment granularity.
*/
export enum SegmentGranularity {
HOUR = 'HOUR',
DAY = 'DAY',
MONTH = 'MONTH',
YEAR = 'YEAR',
HOUR = 'hour',
DAY = 'day',
MONTH = 'month',
YEAR = 'year',
}
const PARTITIONING_TYPE = 'Partitioning type';

13
web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
View File

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`auto-form snapshot matches snapshot 1`] = `
exports[`AutoForm matches snapshot 1`] = `
<div
className="auto-form"
>
@ -132,5 +132,16 @@ exports[`auto-form snapshot matches snapshot 1`] = `
placeholder=""
/>
</Memo(FormGroupWithInfo)>
<Blueprint3.FormGroup
key="more-or-less"
>
<Blueprint3.Button
fill={true}
minimal={true}
onClick={[Function]}
rightIcon="chevron-down"
text="Show more"
/>
</Blueprint3.FormGroup>
</div>
`;

5
web-console/src/components/auto-form/auto-form.spec.tsx
View File

@ -21,7 +21,7 @@ import React from 'react';
import { AutoForm } from './auto-form';
describe('auto-form snapshot', () => {
describe('AutoForm', () => {
it('matches snapshot', () => {
const autoForm = shallow(
<AutoForm
@ -34,6 +34,9 @@ describe('auto-form snapshot', () => {
{ name: 'testFive', type: 'string-array' },
{ name: 'testSix', type: 'json' },
{ name: 'testSeven', type: 'json' },
{ name: 'testNotDefined', type: 'string', defined: false },
{ name: 'testAdvanced', type: 'string', hideInMore: true },
]}
model={String}
onChange={() => {}}

52
web-console/src/components/auto-form/auto-form.tsx
View File

@ -17,6 +17,7 @@
*/
import { Button, ButtonGroup, FormGroup, Intent, NumericInput } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import { deepDelete, deepGet, deepSet } from '../../utils';
@ -54,6 +55,7 @@ export interface Field<M> {
disabled?: Functor<M, boolean>;
defined?: Functor<M, boolean>;
required?: Functor<M, boolean>;
hideInMore?: Functor<M, boolean>;
adjustment?: (model: M) => M;
issueWithValue?: (value: any) => string | undefined;
}
@ -68,7 +70,14 @@ export interface AutoFormProps<M> {
globalAdjustment?: (model: M) => M;
}
export class AutoForm<T extends Record<string, any>> extends React.PureComponent<AutoFormProps<T>> {
export interface AutoFormState {
showMore: boolean;
}
export class AutoForm<T extends Record<string, any>> extends React.PureComponent<
AutoFormProps<T>,
AutoFormState
> {
static REQUIRED_INTENT = Intent.PRIMARY;
static makeLabelName(label: string): string {
@ -138,7 +147,9 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
constructor(props: AutoFormProps<T>) {
super(props);
this.state = {};
this.state = {
showMore: false,
};
}
private fieldChange = (field: Field<T>, newValue: any) => {
@ -391,7 +402,6 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
private renderField = (field: Field<T>) => {
const { model } = this.props;
if (!model) return;
if (!AutoForm.evaluateFunctor(field.defined, model, true)) return;
const label = field.label || AutoForm.makeLabelName(field.name);
return (
@ -415,12 +425,46 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
);
}
renderMoreOrLess() {
const { showMore } = this.state;
return (
<FormGroup key="more-or-less">
<Button
text={showMore ? 'Show less' : 'Show more'}
rightIcon={showMore ? IconNames.CHEVRON_UP : IconNames.CHEVRON_DOWN}
minimal
fill
onClick={() => {
this.setState(({ showMore }) => ({ showMore: !showMore }));
}}
/>
</FormGroup>
);
}
render(): JSX.Element {
const { fields, model, showCustom } = this.props;
const { showMore } = this.state;
let shouldShowMore = false;
const shownFields = fields.filter(field => {
if (AutoForm.evaluateFunctor(field.defined, model, true)) {
if (AutoForm.evaluateFunctor(field.hideInMore, model, false)) {
shouldShowMore = true;
return showMore;
}
return true;
} else {
return false;
}
});
return (
<div className="auto-form">
{model && fields.map(this.renderField)}
{model && shownFields.map(this.renderField)}
{model && showCustom && showCustom(model) && this.renderCustom()}
{shouldShowMore && this.renderMoreOrLess()}
</div>
);
}

10
web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap
View File

@ -17,6 +17,10 @@ exports[`coordinator dynamic config matches snapshot 1`] = `
</Memo(ExternalLink)>
.
</p>
<Memo(FormJsonSelector)
onChange={[Function]}
tab="form"
/>
<AutoForm
fields={
Array [
@ -47,13 +51,11 @@ exports[`coordinator dynamic config matches snapshot 1`] = `
Object {
"defaultValue": false,
"info": <React.Fragment>
Send kill tasks for ALL dataSources if property
Send kill tasks for ALL dataSources if property
<Unknown>
druid.coordinator.kill.on
</Unknown>
is true. If this is set to true then
is true. If this is set to true then
<Unknown>
killDataSourceWhitelist
</Unknown>

198
web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx
View File

@ -16,13 +16,20 @@
* limitations under the License.
*/
import { Code, Intent } from '@blueprintjs/core';
import { Intent } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import axios from 'axios';
import React, { useState } from 'react';
import { SnitchDialog } from '..';
import { AutoForm, ExternalLink } from '../../components';
import {
AutoForm,
ExternalLink,
FormJsonSelector,
FormJsonTabs,
JsonInput,
} from '../../components';
import { COORDINATOR_DYNAMIC_CONFIG_FIELDS, CoordinatorDynamicConfig } from '../../druid-models';
import { useQueryManager } from '../../hooks';
import { getLink } from '../../links';
import { AppToaster } from '../../singletons/toaster';
@ -38,7 +45,8 @@ export const CoordinatorDynamicConfigDialog = React.memo(function CoordinatorDyn
props: CoordinatorDynamicConfigDialogProps,
) {
const { onClose } = props;
const [dynamicConfig, setDynamicConfig] = useState<Record<string, any>>({});
const [currentTab, setCurrentTab] = useState<FormJsonTabs>('form');
const [dynamicConfig, setDynamicConfig] = useState<CoordinatorDynamicConfig>({});
const [historyRecordsState] = useQueryManager<null, any[]>({
processQuery: async () => {
@ -106,180 +114,16 @@ export const CoordinatorDynamicConfigDialog = React.memo(function CoordinatorDyn
</ExternalLink>
.
</p>
<AutoForm
fields={[
{
name: 'maxSegmentsToMove',
type: 'number',
defaultValue: 5,
info: <>The maximum number of segments that can be moved at any given time.</>,
},
{
name: 'balancerComputeThreads',
type: 'number',
defaultValue: 1,
info: (
<>
Thread pool size for computing moving cost of segments in segment balancing.
Consider increasing this if you have a lot of segments and moving segments starts to
get stuck.
</>
),
},
{
name: 'emitBalancingStats',
type: 'boolean',
defaultValue: false,
info: (
<>
Boolean flag for whether or not we should emit balancing stats. This is an expensive
operation.
</>
),
},
{
name: 'killAllDataSources',
type: 'boolean',
defaultValue: false,
info: (
<>
Send kill tasks for ALL dataSources if property{' '}
<Code>druid.coordinator.kill.on</Code> is true. If this is set to true then{' '}
<Code>killDataSourceWhitelist</Code> must not be specified or be empty list.
</>
),
},
{
name: 'killDataSourceWhitelist',
type: 'string-array',
emptyValue: [],
info: (
<>
List of dataSources for which kill tasks are sent if property{' '}
<Code>druid.coordinator.kill.on</Code> is true. This can be a list of
comma-separated dataSources or a JSON array.
</>
),
},
{
name: 'killPendingSegmentsSkipList',
type: 'string-array',
emptyValue: [],
info: (
<>
List of dataSources for which pendingSegments are NOT cleaned up if property{' '}
<Code>druid.coordinator.kill.pendingSegments.on</Code> is true. This can be a list
of comma-separated dataSources or a JSON array.
</>
),
},
{
name: 'maxSegmentsInNodeLoadingQueue',
type: 'number',
defaultValue: 0,
info: (
<>
The maximum number of segments that could be queued for loading to any given server.
This parameter could be used to speed up segments loading process, especially if
there are "slow" nodes in the cluster (with low loading speed) or if too much
segments scheduled to be replicated to some particular node (faster loading could be
preferred to better segments distribution). Desired value depends on segments
loading speed, acceptable replication time and number of nodes. Value 1000 could be
a start point for a rather big cluster. Default value is 0 (loading queue is
unbounded)
</>
),
},
{
name: 'mergeBytesLimit',
type: 'size-bytes',
defaultValue: 524288000,
info: <>The maximum total uncompressed size in bytes of segments to merge.</>,
},
{
name: 'mergeSegmentsLimit',
type: 'number',
defaultValue: 100,
info: <>The maximum number of segments that can be in a single append task.</>,
},
{
name: 'millisToWaitBeforeDeleting',
type: 'number',
defaultValue: 900000,
info: (
<>
How long does the Coordinator need to be active before it can start removing
(marking unused) segments in metadata storage.
</>
),
},
{
name: 'replicantLifetime',
type: 'number',
defaultValue: 15,
info: (
<>
The maximum number of Coordinator runs for a segment to be replicated before we
start alerting.
</>
),
},
{
name: 'replicationThrottleLimit',
type: 'number',
defaultValue: 10,
info: <>The maximum number of segments that can be replicated at one time.</>,
},
{
name: 'decommissioningNodes',
type: 'string-array',
emptyValue: [],
info: (
<>
List of historical services to 'decommission'. Coordinator will not assign new
segments to 'decommissioning' services, and segments will be moved away from them to
be placed on non-decommissioning services at the maximum rate specified by{' '}
<Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code>.
</>
),
},
{
name: 'decommissioningMaxPercentOfMaxSegmentsToMove',
type: 'number',
defaultValue: 70,
info: (
<>
The maximum number of segments that may be moved away from 'decommissioning'
services to non-decommissioning (that is, active) services during one Coordinator
run. This value is relative to the total maximum segment movements allowed during
one run which is determined by <Code>maxSegmentsToMove</Code>. If
<Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code> is 0, segments will
neither be moved from or to 'decommissioning' services, effectively putting them in
a sort of "maintenance" mode that will not participate in balancing or assignment by
load rules. Decommissioning can also become stalled if there are no available active
services to place the segments. By leveraging the maximum percent of decommissioning
segment movements, an operator can prevent active services from overload by
prioritizing balancing, or decrease decommissioning time instead. The value should
be between 0 and 100.
</>
),
},
{
name: 'pauseCoordination',
type: 'boolean',
defaultValue: false,
info: (
<>
Boolean flag for whether or not the coordinator should execute its various duties of
coordinating the cluster. Setting this to true essentially pauses all coordination
work while allowing the API to remain up.
</>
),
},
]}
model={dynamicConfig}
onChange={m => setDynamicConfig(m)}
/>
<FormJsonSelector tab={currentTab} onChange={setCurrentTab} />
{currentTab === 'form' ? (
<AutoForm
fields={COORDINATOR_DYNAMIC_CONFIG_FIELDS}
model={dynamicConfig}
onChange={setDynamicConfig}
/>
) : (
<JsonInput value={dynamicConfig} onChange={setDynamicConfig} height="50vh" />
)}
</SnitchDialog>
);
});

16
web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx
View File

@ -23,6 +23,7 @@ import React, { useState } from 'react';
import { SnitchDialog } from '..';
import { AutoForm, ExternalLink } from '../../components';
import { OVERLORD_DYNAMIC_CONFIG_FIELDS, OverlordDynamicConfig } from '../../druid-models';
import { useQueryManager } from '../../hooks';
import { getLink } from '../../links';
import { AppToaster } from '../../singletons/toaster';
@ -38,7 +39,7 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo
props: OverlordDynamicConfigDialogProps,
) {
const { onClose } = props;
const [dynamicConfig, setDynamicConfig] = useState<Record<string, any>>({});
const [dynamicConfig, setDynamicConfig] = useState<OverlordDynamicConfig>({});
const [historyRecordsState] = useQueryManager<null, any[]>({
processQuery: async () => {
@ -108,18 +109,9 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo
.
</p>
<AutoForm
fields={[
{
name: 'selectStrategy',
type: 'json',
},
{
name: 'autoScaler',
type: 'json',
},
]}
fields={OVERLORD_DYNAMIC_CONFIG_FIELDS}
model={dynamicConfig}
onChange={m => setDynamicConfig(m)}
onChange={setDynamicConfig}
/>
</SnitchDialog>
);

4
web-console/src/druid-models/__snapshots__/ingestion-spec.spec.ts.snap
View File

@ -13,9 +13,9 @@ Object {
],
},
"granularitySpec": Object {
"queryGranularity": "HOUR",
"queryGranularity": "hour",
"rollup": true,
"segmentGranularity": "DAY",
"segmentGranularity": "day",
"type": "uniform",
},
"metricsSpec": Array [

208
web-console/src/druid-models/coordinator-dynamic-config.tsx Normal file
View File

@ -0,0 +1,208 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Code } from '@blueprintjs/core';
import React from 'react';
import { Field } from '../components';
export interface CoordinatorDynamicConfig {
maxSegmentsToMove?: number;
balancerComputeThreads?: number;
emitBalancingStats?: boolean;
killAllDataSources?: boolean;
killDataSourceWhitelist?: string[];
killPendingSegmentsSkipList?: string[];
maxSegmentsInNodeLoadingQueue?: number;
mergeBytesLimit?: number;
mergeSegmentsLimit?: number;
millisToWaitBeforeDeleting?: number;
replicantLifetime?: number;
replicationThrottleLimit?: number;
decommissioningNodes?: string[];
decommissioningMaxPercentOfMaxSegmentsToMove?: number;
pauseCoordination?: boolean;
}
export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field<CoordinatorDynamicConfig>[] = [
{
name: 'maxSegmentsToMove',
type: 'number',
defaultValue: 5,
info: <>The maximum number of segments that can be moved at any given time.</>,
},
{
name: 'balancerComputeThreads',
type: 'number',
defaultValue: 1,
info: (
<>
Thread pool size for computing moving cost of segments in segment balancing. Consider
increasing this if you have a lot of segments and moving segments starts to get stuck.
</>
),
},
{
name: 'emitBalancingStats',
type: 'boolean',
defaultValue: false,
info: (
<>
Boolean flag for whether or not we should emit balancing stats. This is an expensive
operation.
</>
),
},
{
name: 'killAllDataSources',
type: 'boolean',
defaultValue: false,
info: (
<>
Send kill tasks for ALL dataSources if property <Code>druid.coordinator.kill.on</Code> is
true. If this is set to true then <Code>killDataSourceWhitelist</Code> must not be specified
or be empty list.
</>
),
},
{
name: 'killDataSourceWhitelist',
type: 'string-array',
emptyValue: [],
info: (
<>
List of dataSources for which kill tasks are sent if property{' '}
<Code>druid.coordinator.kill.on</Code> is true. This can be a list of comma-separated
dataSources or a JSON array.
</>
),
},
{
name: 'killPendingSegmentsSkipList',
type: 'string-array',
emptyValue: [],
info: (
<>
List of dataSources for which pendingSegments are NOT cleaned up if property{' '}
<Code>druid.coordinator.kill.pendingSegments.on</Code> is true. This can be a list of
comma-separated dataSources or a JSON array.
</>
),
},
{
name: 'maxSegmentsInNodeLoadingQueue',
type: 'number',
defaultValue: 0,
info: (
<>
The maximum number of segments that could be queued for loading to any given server. This
parameter could be used to speed up segments loading process, especially if there are "slow"
nodes in the cluster (with low loading speed) or if too much segments scheduled to be
replicated to some particular node (faster loading could be preferred to better segments
distribution). Desired value depends on segments loading speed, acceptable replication time
and number of nodes. Value 1000 could be a start point for a rather big cluster. Default
value is 0 (loading queue is unbounded)
</>
),
},
{
name: 'mergeBytesLimit',
type: 'size-bytes',
defaultValue: 524288000,
info: <>The maximum total uncompressed size in bytes of segments to merge.</>,
},
{
name: 'mergeSegmentsLimit',
type: 'number',
defaultValue: 100,
info: <>The maximum number of segments that can be in a single append task.</>,
},
{
name: 'millisToWaitBeforeDeleting',
type: 'number',
defaultValue: 900000,
info: (
<>
How long does the Coordinator need to be active before it can start removing (marking
unused) segments in metadata storage.
</>
),
},
{
name: 'replicantLifetime',
type: 'number',
defaultValue: 15,
info: (
<>
The maximum number of Coordinator runs for a segment to be replicated before we start
alerting.
</>
),
},
{
name: 'replicationThrottleLimit',
type: 'number',
defaultValue: 10,
info: <>The maximum number of segments that can be replicated at one time.</>,
},
{
name: 'decommissioningNodes',
type: 'string-array',
emptyValue: [],
info: (
<>
List of historical services to 'decommission'. Coordinator will not assign new segments to
'decommissioning' services, and segments will be moved away from them to be placed on
non-decommissioning services at the maximum rate specified by{' '}
<Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code>.
</>
),
},
{
name: 'decommissioningMaxPercentOfMaxSegmentsToMove',
type: 'number',
defaultValue: 70,
info: (
<>
The maximum number of segments that may be moved away from 'decommissioning' services to
non-decommissioning (that is, active) services during one Coordinator run. This value is
relative to the total maximum segment movements allowed during one run which is determined
by <Code>maxSegmentsToMove</Code>. If
<Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code> is 0, segments will neither be
moved from or to 'decommissioning' services, effectively putting them in a sort of
"maintenance" mode that will not participate in balancing or assignment by load rules.
Decommissioning can also become stalled if there are no available active services to place
the segments. By leveraging the maximum percent of decommissioning segment movements, an
operator can prevent active services from overload by prioritizing balancing, or decrease
decommissioning time instead. The value should be between 0 and 100.
</>
),
},
{
name: 'pauseCoordination',
type: 'boolean',
defaultValue: false,
info: (
<>
Boolean flag for whether or not the coordinator should execute its various duties of
coordinating the cluster. Setting this to true essentially pauses all coordination work
while allowing the API to remain up.
</>
),
},
];

2
web-console/src/druid-models/index.ts
View File

@ -29,3 +29,5 @@ export * from './filter';
export * from './dimension-spec';
export * from './metric-spec';
export * from './ingestion-spec';
export * from './coordinator-dynamic-config';
export * from './overlord-dynamic-config';

12
web-console/src/druid-models/ingestion-spec.spec.ts
View File

@ -45,8 +45,8 @@ describe('ingestion-spec', () => {
dataSource: 'wikipedia',
granularitySpec: {
type: 'uniform',
segmentGranularity: 'DAY',
queryGranularity: 'HOUR',
segmentGranularity: 'day',
queryGranularity: 'hour',
rollup: true,
},
parser: {
@ -183,8 +183,8 @@ describe('spec utils', () => {
dataSource: 'wikipedia',
granularitySpec: {
type: 'uniform',
segmentGranularity: 'DAY',
queryGranularity: 'HOUR',
segmentGranularity: 'day',
queryGranularity: 'hour',
},
timestampSpec: {
column: 'timestamp',
@ -219,9 +219,9 @@ describe('spec utils', () => {
],
},
"granularitySpec": Object {
"queryGranularity": "HOUR",
"queryGranularity": "hour",
"rollup": true,
"segmentGranularity": "DAY",
"segmentGranularity": "day",
"type": "uniform",
},
"metricsSpec": Array [

240
web-console/src/druid-models/ingestion-spec.tsx
View File

@ -47,7 +47,7 @@ import {
getMetricSpecSingleFieldName,
MetricSpec,
} from './metric-spec';
import { PLACEHOLDER_TIMESTAMP_SPEC, TimestampSpec } from './timestamp-spec';
import { TimestampSpec } from './timestamp-spec';
import { TransformSpec } from './transform-spec';
export const MAX_INLINE_DATA_LENGTH = 65536;
@ -475,6 +475,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
label: 'Dimensions',
type: 'string-array',
placeholder: '(optional)',
hideInMore: true,
info: (
<p>
The list of dimensions to select. If left empty, no dimensions are returned. If left
@ -487,6 +488,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
label: 'Metrics',
type: 'string-array',
placeholder: '(optional)',
hideInMore: true,
info: (
<p>
The list of metrics to select. If left empty, no metrics are returned. If left null or
@ -499,6 +501,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
label: 'Filter',
type: 'json',
placeholder: '(optional)',
hideInMore: true,
info: (
<p>
The{' '}
@ -983,45 +986,6 @@ export function getIoConfigTuningFormFields(
</>
),
},
{
name: 'inputSource.maxCacheCapacityBytes',
label: 'Max cache capacity bytes',
type: 'number',
defaultValue: 1073741824,
info: (
<>
<p>
Maximum size of the cache space in bytes. 0 means disabling cache. Cached files are
not removed until the ingestion task completes.
</p>
</>
),
},
{
name: 'inputSource.maxFetchCapacityBytes',
label: 'Max fetch capacity bytes',
type: 'number',
defaultValue: 1073741824,
info: (
<>
<p>
Maximum size of the fetch space in bytes. 0 means disabling prefetch. Prefetched
files are removed immediately once they are read.
</p>
</>
),
},
{
name: 'inputSource.prefetchTriggerBytes',
label: 'Prefetch trigger bytes',
type: 'number',
placeholder: 'maxFetchCapacityBytes / 2',
info: (
<>
<p>Threshold to trigger prefetching the objects.</p>
</>
),
},
];
case 'index_parallel:local':
@ -1420,6 +1384,7 @@ export function invalidTuningConfig(tuningConfig: TuningConfig, intervals: any):
export function getPartitionRelatedTuningSpecFormFields(
specType: IngestionType,
dimensionSuggestions: string[] | undefined,
): Field<TuningConfig>[] {
switch (specType) {
case 'index_parallel':
@ -1437,6 +1402,10 @@ export function getPartitionRelatedTuningSpecFormFields(
single dimension). For best-effort rollup, you should use <Code>dynamic</Code>.
</p>
),
adjustment: (t: TuningConfig) => {
if (!Array.isArray(dimensionSuggestions) || !dimensionSuggestions.length) return t;
return deepSet(t, 'partitionsSpec.partitionDimension', dimensionSuggestions[0]);
},
},
// partitionsSpec type: dynamic
{
@ -1460,6 +1429,8 @@ export function getPartitionRelatedTuningSpecFormFields(
name: 'partitionsSpec.targetRowsPerSegment',
label: 'Target rows per segment',
type: 'number',
zeroMeansUndefined: true,
defaultValue: 5000000,
defined: (t: TuningConfig) =>
deepGet(t, 'partitionsSpec.type') === 'hashed' &&
!deepGet(t, 'partitionsSpec.numShards'),
@ -1481,6 +1452,8 @@ export function getPartitionRelatedTuningSpecFormFields(
name: 'partitionsSpec.numShards',
label: 'Num shards',
type: 'number',
zeroMeansUndefined: true,
hideInMore: true,
defined: (t: TuningConfig) =>
deepGet(t, 'partitionsSpec.type') === 'hashed' &&
!deepGet(t, 'partitionsSpec.targetRowsPerSegment'),
@ -1502,6 +1475,7 @@ export function getPartitionRelatedTuningSpecFormFields(
name: 'partitionsSpec.partitionDimensions',
label: 'Partition dimensions',
type: 'string-array',
placeholder: '(all dimensions)',
defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'hashed',
info: <p>The dimensions to partition on. Leave blank to select all dimensions.</p>,
},
@ -1512,7 +1486,19 @@ export function getPartitionRelatedTuningSpecFormFields(
type: 'string',
defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'single_dim',
required: true,
info: <p>The dimension to partition on.</p>,
suggestions: dimensionSuggestions,
info: (
<>
<p>The dimension to partition on.</p>
<p>
This should be the first dimension in your schema which would make it first in the
sort order. As{' '}
<ExternalLink href={`${getLink('DOCS')}/ingestion/index.html#why-partition`}>
Partitioning and sorting are best friends!
</ExternalLink>
</p>
</>
),
},
{
name: 'partitionsSpec.targetRowsPerSegment',
@ -1603,6 +1589,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
defaultValue: 3,
defined: (t: TuningConfig) => t.type === 'index_parallel',
hideInMore: true,
info: <>Maximum number of retries on task failures.</>,
},
{
@ -1610,8 +1597,36 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
defaultValue: 1000,
defined: (t: TuningConfig) => t.type === 'index_parallel',
hideInMore: true,
info: <>Polling period in milliseconds to check running task statuses.</>,
},
{
name: 'totalNumMergeTasks',
type: 'number',
defaultValue: 10,
min: 1,
defined: (t: TuningConfig) =>
Boolean(
t.type === 'index_parallel' &&
oneOf(deepGet(t, 'partitionsSpec.type'), 'hashed', 'single_dim'),
),
info: <>Number of tasks to merge partial segments after shuffle.</>,
},
{
name: 'maxNumSegmentsToMerge',
type: 'number',
defaultValue: 100,
defined: (t: TuningConfig) =>
Boolean(
t.type === 'index_parallel' &&
oneOf(deepGet(t, 'partitionsSpec.type'), 'hashed', 'single_dim'),
),
info: (
<>
Max limit for the number of segments a single task can merge at the same time after shuffle.
</>
),
},
{
name: 'maxRowsInMemory',
type: 'number',
@ -1624,33 +1639,6 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
placeholder: 'Default: 1/6 of max JVM memory',
info: <>Used in determining when intermediate persists to disk should occur.</>,
},
{
name: 'totalNumMergeTasks',
type: 'number',
defaultValue: 10,
min: 1,
defined: (t: TuningConfig) =>
Boolean(
t.type === 'index_parallel' &&
oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'),
),
info: <>Number of tasks to merge partial segments after shuffle.</>,
},
{
name: 'maxNumSegmentsToMerge',
type: 'number',
defaultValue: 100,
defined: (t: TuningConfig) =>
Boolean(
t.type === 'index_parallel' &&
oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'),
),
info: (
<>
Max limit for the number of segments a single task can merge at the same time after shuffle.
</>
),
},
{
name: 'resetOffsetAutomatically',
type: 'boolean',
@ -1663,6 +1651,19 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
</>
),
},
{
name: 'skipSequenceNumberAvailabilityCheck',
type: 'boolean',
defaultValue: false,
defined: (t: TuningConfig) => t.type === 'kinesis',
info: (
<>
Whether to enable checking if the current sequence number is still available in a particular
Kinesis shard. If set to false, the indexing task will attempt to reset the current sequence
number (or not), depending on the value of <Code>resetOffsetAutomatically</Code>.
</>
),
},
{
name: 'intermediatePersistPeriod',
type: 'duration',
@ -1686,6 +1687,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
{
name: 'maxPendingPersists',
type: 'number',
hideInMore: true,
info: (
<>
Maximum number of persists that can be pending but not started. If this limit would be
@ -1698,6 +1700,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
name: 'pushTimeout',
type: 'number',
defaultValue: 0,
hideInMore: true,
info: (
<>
Milliseconds to wait for pushing segments. It must be >= 0, where 0 means to wait forever.
@ -1709,6 +1712,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
defaultValue: 0,
defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
hideInMore: true,
info: <>Milliseconds to wait for segment handoff. 0 means to wait forever.</>,
},
{
@ -1717,6 +1721,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'string',
defaultValue: 'roaring',
suggestions: ['concise', 'roaring'],
hideInMore: true,
info: <>Compression format for bitmap indexes.</>,
},
{
@ -1725,6 +1730,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'string',
defaultValue: 'lz4',
suggestions: ['lz4', 'lzf', 'uncompressed'],
hideInMore: true,
info: <>Compression format for dimension columns.</>,
},
{
@ -1733,6 +1739,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'string',
defaultValue: 'lz4',
suggestions: ['lz4', 'lzf', 'uncompressed'],
hideInMore: true,
info: <>Compression format for primitive type metric columns.</>,
},
{
@ -1741,6 +1748,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'string',
defaultValue: 'longs',
suggestions: ['longs', 'auto'],
hideInMore: true,
info: (
<>
Encoding format for long-typed columns. Applies regardless of whether they are dimensions or
@ -1755,6 +1763,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'duration',
defaultValue: 'PT10S',
defined: (t: TuningConfig) => t.type === 'index_parallel',
hideInMore: true,
info: <>Timeout for reporting the pushed segments in worker tasks.</>,
},
{
@ -1762,6 +1771,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
defaultValue: 5,
defined: (t: TuningConfig) => t.type === 'index_parallel',
hideInMore: true,
info: <>Retries for reporting the pushed segments in worker tasks.</>,
},
{
@ -1778,6 +1788,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
placeholder: 'min(10, taskCount * replicas)',
defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
hideInMore: true,
info: <>The number of threads that will be used for communicating with indexing tasks.</>,
},
{
@ -1785,6 +1796,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
defaultValue: 8,
defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
hideInMore: true,
info: (
<>
The number of times HTTP requests to indexing tasks will be retried before considering tasks
@ -1804,6 +1816,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'duration',
defaultValue: 'PT80S',
defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
hideInMore: true,
info: (
<>
How long to wait for the supervisor to attempt a graceful shutdown of tasks before exiting.
@ -1839,6 +1852,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
defaultValue: 5000,
defined: (t: TuningConfig) => t.type === 'kinesis',
hideInMore: true,
info: (
<>
Length of time in milliseconds to wait for space to become available in the buffer before
@ -1848,6 +1862,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
},
{
name: 'recordBufferFullWait',
hideInMore: true,
type: 'number',
defaultValue: 5000,
defined: (t: TuningConfig) => t.type === 'kinesis',
@ -1863,6 +1878,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
defaultValue: 60000,
defined: (t: TuningConfig) => t.type === 'kinesis',
hideInMore: true,
info: (
<>
Length of time in milliseconds to wait for Kinesis to return the earliest or latest sequence
@ -1877,6 +1893,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
placeholder: 'max(1, {numProcessors} - 1)',
defined: (t: TuningConfig) => t.type === 'kinesis',
hideInMore: true,
info: (
<>
Size of the pool of threads fetching data from Kinesis. There is no benefit in having more
@ -1889,6 +1906,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
type: 'number',
defaultValue: 100,
defined: (t: TuningConfig) => t.type === 'kinesis',
hideInMore: true,
info: (
<>
The maximum number of records/events to be fetched from buffer per poll. The actual maximum
@ -1896,6 +1914,29 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
</>
),
},
{
name: 'repartitionTransitionDuration',
type: 'duration',
defaultValue: 'PT2M',
defined: (t: TuningConfig) => t.type === 'kinesis',
hideInMore: true,
info: (
<>
<p>
When shards are split or merged, the supervisor will recompute shard, task group mappings,
and signal any running tasks created under the old mappings to stop early at{' '}
<Code>(current time + repartitionTransitionDuration)</Code>. Stopping the tasks early
allows Druid to begin reading from the new shards more quickly.
</p>
<p>
The repartition transition wait time controlled by this property gives the stream
additional time to write records to the new shards after the split/merge, which helps
avoid the issues with empty shard handling described at
<ExternalLink href="https://github.com/apache/druid/issues/7600">#7600</ExternalLink>.
</p>
</>
),
},
];
export function getTuningSpecFormFields() {
@ -1932,39 +1973,39 @@ export function updateIngestionType(
if (inputSourceType) {
newSpec = deepSet(newSpec, 'spec.ioConfig.inputSource', { type: inputSourceType });
if (inputSourceType === 'local') {
newSpec = deepSet(newSpec, 'spec.ioConfig.inputSource.filter', '*');
}
}
if (!deepGet(spec, 'spec.dataSchema.dataSource')) {
newSpec = deepSet(newSpec, 'spec.dataSchema.dataSource', 'new-data-source');
}
if (!deepGet(spec, 'spec.dataSchema.granularitySpec')) {
const granularitySpec: GranularitySpec = {
type: 'uniform',
queryGranularity: 'HOUR',
};
if (ingestionType !== 'index_parallel') {
granularitySpec.segmentGranularity = 'HOUR';
}
newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec', granularitySpec);
}
if (!deepGet(spec, 'spec.dataSchema.timestampSpec')) {
newSpec = deepSet(newSpec, 'spec.dataSchema.timestampSpec', PLACEHOLDER_TIMESTAMP_SPEC);
}
if (!deepGet(spec, 'spec.dataSchema.dimensionsSpec')) {
newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec', {});
}
return newSpec;
}
export function issueWithSampleData(sampleData: string[]): JSX.Element | undefined {
if (sampleData.length) {
const firstData = sampleData[0];
if (firstData === '{') {
return (
<>
This data looks like regular JSON object. For Druid to parse a text file it must have one
row per event. Maybe look at{' '}
<ExternalLink href="http://ndjson.org/">newline delimited JSON</ExternalLink> instead.
</>
);
}
if (oneOf(firstData, '[', '[]')) {
return (
<>
This data looks like a multi-line JSON array. For Druid to parse a text file it must have
one row per event. Maybe look at{' '}
<ExternalLink href="http://ndjson.org/">newline delimited JSON</ExternalLink> instead.
</>
);
}
}
return;
}
export function fillInputFormat(spec: IngestionSpec, sampleData: string[]): IngestionSpec {
return deepSet(spec, 'spec.ioConfig.inputFormat', guessInputFormat(sampleData));
}
@ -2080,7 +2121,8 @@ export function updateSchemaWithSample(
let newSpec = spec;
if (dimensionMode === 'auto-detect') {
newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions', []);
newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions');
newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensionExclusions', []);
} else {
newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.dimensionExclusions');
@ -2091,14 +2133,14 @@ export function updateSchemaWithSample(
}
if (rollup) {
newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'HOUR');
newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'hour');
const metrics = getMetricSpecs(headerAndRows, typeHints);
if (metrics) {
newSpec = deepSet(newSpec, 'spec.dataSchema.metricsSpec', metrics);
}
} else {
newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'NONE');
newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'none');
newSpec = deepDelete(newSpec, 'spec.dataSchema.metricsSpec');
}

35
web-console/src/druid-models/input-format.tsx
View File

@ -16,6 +16,7 @@
* limitations under the License.
*/
import { Code } from '@blueprintjs/core';
import React from 'react';
import { AutoForm, ExternalLink, Field } from '../components';
@ -68,12 +69,6 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
required: true,
defined: (p: InputFormat) => p.type === 'javascript',
},
{
name: 'findColumnsFromHeader',
type: 'boolean',
required: true,
defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv'),
},
{
name: 'skipHeaderRows',
type: 'number',
@ -82,9 +77,22 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
min: 0,
info: (
<>
If both skipHeaderRows and hasHeaderRow options are set, skipHeaderRows is first applied.
For example, if you set skipHeaderRows to 2 and hasHeaderRow to true, Druid will skip the
first two lines and then extract column information from the third line.
If this is set, skip the first <Code>skipHeaderRows</Code> rows from each file.
</>
),
},
{
name: 'findColumnsFromHeader',
type: 'boolean',
required: true,
defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv'),
info: (
<>
If this is set, find the column names from the header row. Note that
<Code>skipHeaderRows</Code> will be applied before finding column names from the header. For
example, if you set <Code>skipHeaderRows</Code> to 2 and <Code>findColumnsFromHeader</Code>{' '}
to true, the task will skip the first two lines and then extract column information from the
third line.
</>
),
},
@ -93,7 +101,13 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
type: 'string-array',
required: true,
defined: (p: InputFormat) =>
(oneOf(p.type, 'csv', 'tsv') && !p.findColumnsFromHeader) || p.type === 'regex',
(oneOf(p.type, 'csv', 'tsv') && p.findColumnsFromHeader === false) || p.type === 'regex',
info: (
<>
Specifies the columns of the data. The columns should be in the same order with the columns
of your data.
</>
),
},
{
name: 'delimiter',
@ -106,6 +120,7 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
name: 'listDelimiter',
type: 'string',
defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv', 'regex'),
placeholder: '(optional, default = ctrl+A)',
info: <>A custom delimiter for multi-value dimensions.</>,
},
{

35
web-console/src/druid-models/overlord-dynamic-config.tsx Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Field } from '../components';
export interface OverlordDynamicConfig {
selectStrategy?: Record<string, any>;
autoScaler?: Record<string, any>;
}
export const OVERLORD_DYNAMIC_CONFIG_FIELDS: Field<OverlordDynamicConfig>[] = [
{
name: 'selectStrategy',
type: 'json',
},
{
name: 'autoScaler',
type: 'json',
},
];

7
web-console/src/druid-models/timestamp-spec.tsx
View File

@ -123,7 +123,7 @@ export const TIMESTAMP_SPEC_FIELDS: Field<TimestampSpec>[] = [
],
info: (
<p>
Please specify your timestamp format by using the suggestions menu or typing in a{' '}
Specify your timestamp format by using the suggestions menu or typing in a{' '}
<ExternalLink href="https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html">
format string
</ExternalLink>
@ -135,7 +135,10 @@ export const TIMESTAMP_SPEC_FIELDS: Field<TimestampSpec>[] = [
name: 'missingValue',
type: 'string',
placeholder: '(optional)',
info: <p>This value will be used if the specified column can not be found.</p>,
info: (
<p>Specify a static value for cases when the source time column is missing or is null.</p>
),
suggestions: ['2020-01-01T00:00:00Z'],
},
];

1
web-console/src/druid-models/transform-spec.tsx
View File

@ -75,6 +75,7 @@ export function getTimestampExpressionFields(transforms: Transform[]): Field<Tra
`timestamp_parse(concat("date", ' ', "time"))`,
`timestamp_parse(concat("date", ' ', "time"), 'M/d/yyyy H:mm:ss')`,
`timestamp_parse(concat("year", '-', "month", '-', "day"))`,
`timestamp_parse("local_time", 'yyyy-MM-dd HH:mm:ss', "timezone")`,
],
info: (
<>

10
web-console/src/utils/general.spec.ts
View File

@ -24,6 +24,7 @@ import {
formatMegabytes,
formatMillions,
formatPercent,
moveElement,
sortWithPrefixSuffix,
sqlQueryCustomTableFilter,
swapElements,
@ -96,6 +97,15 @@ describe('general', () => {
});
});
describe('moveElement', () => {
it('moves items in an array', () => {
expect(moveElement(['a', 'b', 'c'], 2, 0)).toEqual(['c', 'a', 'b']);
expect(moveElement(['a', 'b', 'c'], 1, 1)).toEqual(['a', 'b', 'c']);
expect(moveElement(['F', 'B', 'W', 'B'], 2, 1)).toEqual(['F', 'W', 'B', 'B']);
expect(moveElement([1, 2, 3], 2, 1)).toEqual([1, 3, 2]);
});
});
describe('formatInteger', () => {
it('works', () => {
expect(formatInteger(10000)).toEqual('10,000');

27
web-console/src/utils/general.tsx
View File

@ -306,7 +306,7 @@ export function sortWithPrefixSuffix(
things: readonly string[],
prefix: readonly string[],
suffix: readonly string[],
cmp: null | ((a: string, b: string) => number),
cmp?: (a: string, b: string) => number,
): string[] {
const pre = uniq(prefix.filter(x => things.includes(x)));
const mid = things.filter(x => !prefix.includes(x) && !suffix.includes(x));
@ -356,3 +356,28 @@ export function swapElements<T>(items: readonly T[], indexA: number, indexB: num
newItems[indexB] = t;
return newItems;
}
export function moveElement<T>(items: readonly T[], fromIndex: number, toIndex: number): T[] {
const indexDiff = fromIndex - toIndex;
if (indexDiff > 0) {
// move left
return [
...items.slice(0, toIndex),
items[fromIndex],
...items.slice(toIndex, fromIndex),
...items.slice(fromIndex + 1, items.length),
];
} else if (indexDiff < 0) {
// move right
const targetIndex = toIndex + 1;
return [
...items.slice(0, fromIndex),
...items.slice(fromIndex + 1, targetIndex),
items[fromIndex],
...items.slice(targetIndex, items.length),
];
} else {
// do nothing
return items.slice();
}
}

3
web-console/src/utils/query-manager.tsx
View File

@ -95,6 +95,7 @@ export class QueryManager<Q, R> {
this.setState(
new QueryState<R>({
data,
lastData: this.state.getSomeData(),
}),
);
},
@ -107,6 +108,7 @@ export class QueryManager<Q, R> {
this.setState(
new QueryState<R>({
error: e,
lastData: this.state.getSomeData(),
}),
);
},
@ -119,6 +121,7 @@ export class QueryManager<Q, R> {
this.setState(
new QueryState<R>({
loading: true,
lastData: this.state.getSomeData(),
}),
);

15
web-console/src/utils/query-state.ts
View File

@ -18,6 +18,13 @@
export type QueryStateState = 'init' | 'loading' | 'data' | 'error';
export interface QueryStateOptions<T, E extends Error = Error> {
loading?: boolean;
error?: E;
data?: T;
lastData?: T;
}
export class QueryState<T, E extends Error = Error> {
static INIT: QueryState<any> = new QueryState({});
static LOADING: QueryState<any> = new QueryState({ loading: true });
@ -25,8 +32,9 @@ export class QueryState<T, E extends Error = Error> {
public state: QueryStateState = 'init';
public error?: E;
public data?: T;
public lastData?: T;
constructor(opts: { loading?: boolean; error?: E; data?: T }) {
constructor(opts: QueryStateOptions<T, E>) {
const hasData = typeof opts.data !== 'undefined';
if (typeof opts.error !== 'undefined') {
if (hasData) {
@ -43,6 +51,7 @@ export class QueryState<T, E extends Error = Error> {
this.state = opts.loading ? 'loading' : 'init';
}
}
this.lastData = opts.lastData;
}
isInit(): boolean {
@ -71,4 +80,8 @@ export class QueryState<T, E extends Error = Error> {
const { data } = this;
return Boolean(data && Array.isArray(data) && data.length === 0);
}
getSomeData(): T | undefined {
return this.data || this.lastData;
}
}

54
web-console/src/utils/sampler.ts
View File

@ -139,33 +139,41 @@ export function applyCache(sampleSpec: SampleSpec, cacheRows: CacheRows) {
return sampleSpec;
}
export function headerFromSampleResponse(
sampleResponse: SampleResponse,
ignoreColumn?: string,
columnOrder?: string[],
): string[] {
export interface HeaderFromSampleResponseOptions {
sampleResponse: SampleResponse;
ignoreTimeColumn?: boolean;
columnOrder?: string[];
suffixColumnOrder?: string[];
}
export function headerFromSampleResponse(options: HeaderFromSampleResponseOptions): string[] {
const { sampleResponse, ignoreTimeColumn, columnOrder, suffixColumnOrder } = options;
let columns = sortWithPrefixSuffix(
dedupe(sampleResponse.data.flatMap(s => (s.parsed ? Object.keys(s.parsed) : []))).sort(),
columnOrder || ['__time'],
[],
suffixColumnOrder || [],
alphanumericCompare,
);
if (ignoreColumn) {
columns = columns.filter(c => c !== ignoreColumn);
if (ignoreTimeColumn) {
columns = columns.filter(c => c !== '__time');
}
return columns;
}
export interface HeaderAndRowsFromSampleResponseOptions extends HeaderFromSampleResponseOptions {
parsedOnly?: boolean;
}
export function headerAndRowsFromSampleResponse(
sampleResponse: SampleResponse,
ignoreColumn?: string,
columnOrder?: string[],
parsedOnly = false,
options: HeaderAndRowsFromSampleResponseOptions,
): HeaderAndRows {
const { sampleResponse, parsedOnly } = options;
return {
header: headerFromSampleResponse(sampleResponse, ignoreColumn, columnOrder),
header: headerFromSampleResponse(options),
rows: parsedOnly ? sampleResponse.data.filter((d: any) => d.parsed) : sampleResponse.data,
};
}
@ -446,11 +454,11 @@ export async function sampleForTransform(
);
specialDimensionSpec.dimensions = dedupe(
headerFromSampleResponse(
sampleResponseHack,
'__time',
['__time'].concat(inputFormatColumns),
).concat(transforms.map(t => t.name)),
headerFromSampleResponse({
sampleResponse: sampleResponseHack,
ignoreTimeColumn: true,
columnOrder: ['__time'].concat(inputFormatColumns),
}).concat(transforms.map(t => t.name)),
);
}
@ -505,11 +513,11 @@ export async function sampleForFilter(
);
specialDimensionSpec.dimensions = dedupe(
headerFromSampleResponse(
sampleResponseHack,
'__time',
['__time'].concat(inputFormatColumns),
).concat(transforms.map(t => t.name)),
headerFromSampleResponse({
sampleResponse: sampleResponseHack,
ignoreTimeColumn: true,
columnOrder: ['__time'].concat(inputFormatColumns),
}).concat(transforms.map(t => t.name)),
);
}

15
web-console/src/utils/utils.spec.ts
View File

@ -41,8 +41,8 @@ describe('utils', () => {
dataSource: 'wikipedia',
granularitySpec: {
type: 'uniform',
segmentGranularity: 'DAY',
queryGranularity: 'HOUR',
segmentGranularity: 'day',
queryGranularity: 'hour',
},
timestampSpec: {
column: 'timestamp',
@ -56,8 +56,11 @@ describe('utils', () => {
// const cacheRows: CacheRows = [{ make: 'Honda', model: 'Civic' }, { make: 'BMW', model: 'M3' }];
it('spec-utils headerFromSampleResponse', () => {
expect(headerFromSampleResponse({ data: [{ input: { a: 1 }, parsed: { a: 1 } }] }))
.toMatchInlineSnapshot(`
expect(
headerFromSampleResponse({
sampleResponse: { data: [{ input: { a: 1 }, parsed: { a: 1 } }] },
}),
).toMatchInlineSnapshot(`
Array [
"a",
]
@ -86,8 +89,8 @@ describe('utils', () => {
"dataSource": "wikipedia",
"dimensionsSpec": Object {},
"granularitySpec": Object {
"queryGranularity": "HOUR",
"segmentGranularity": "DAY",
"queryGranularity": "hour",
"segmentGranularity": "day",
"type": "uniform",
},
"timestampSpec": Object {

18
web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
View File

@ -46,15 +46,6 @@ exports[`load data view matches snapshot 1`] = `
onClick={[Function]}
text="Parse data"
/>
<Blueprint3.Button
active={false}
className="timestamp"
disabled={true}
icon={false}
key="timestamp"
onClick={[Function]}
text="Parse time"
/>
</Blueprint3.ButtonGroup>
</div>
<div
@ -69,6 +60,15 @@ exports[`load data view matches snapshot 1`] = `
<Blueprint3.ButtonGroup
className="step-nav-l2"
>
<Blueprint3.Button
active={false}
className="timestamp"
disabled={true}
icon={false}
key="timestamp"
onClick={[Function]}
text="Parse time"
/>
<Blueprint3.Button
active={false}
className="transform"

191
web-console/src/views/load-data-view/info-messages.tsx Normal file
View File

@ -0,0 +1,191 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Callout, Code } from '@blueprintjs/core';
import React from 'react';
import { ExternalLink } from '../../components';
import { DimensionMode, getIngestionDocLink, IngestionSpec } from '../../druid-models';
import { getLink } from '../../links';
import { LearnMore } from './learn-more/learn-more';
export interface ConnectMessageProps {
inlineMode: boolean;
spec: IngestionSpec;
}
export const ConnectMessage = React.memo(function ConnectMessage(props: ConnectMessageProps) {
const { inlineMode, spec } = props;
return (
<Callout className="intro">
<p>
Druid ingests raw data and converts it into a custom,{' '}
<ExternalLink href={`${getLink('DOCS')}/design/segments.html`}>indexed format</ExternalLink>{' '}
that is optimized for analytic queries.
</p>
{inlineMode ? (
<>
<p>To get started, please paste some data in the box to the left.</p>
<p>Click "Apply" to verify your data with Druid.</p>
</>
) : (
<p>To get started, please specify what data you want to ingest.</p>
)}
<LearnMore href={getIngestionDocLink(spec)} />
</Callout>
);
});
export interface ParserMessageProps {
canFlatten: boolean;
}
export const ParserMessage = React.memo(function ParserMessage(props: ParserMessageProps) {
const { canFlatten } = props;
return (
<Callout className="intro">
<p>
Druid requires flat data (non-nested, non-hierarchical). Each row should represent a
discrete event.
</p>
{canFlatten && (
<p>
If you have nested data, you can{' '}
<ExternalLink href={`${getLink('DOCS')}/ingestion/index.html#flattenspec`}>
flatten
</ExternalLink>{' '}
it here. If the provided flattening capabilities are not sufficient, please pre-process
your data before ingesting it into Druid.
</p>
)}
<p>Ensure that your data appears correctly in a row/column orientation.</p>
<LearnMore href={`${getLink('DOCS')}/ingestion/data-formats.html`} />
</Callout>
);
});
export const TimestampMessage = React.memo(function TimestampMessage() {
return (
<Callout className="intro">
<p>
Druid partitions data based on the primary time column of your data. This column is stored
internally in Druid as <Code>__time</Code>.
</p>
<p>Configure how to define the time column for this data.</p>
<p>
If your data does not have a time column, you can select <Code>None</Code> to use a
placeholder value. If the time information is spread across multiple columns you can combine
them into one by selecting <Code>Expression</Code> and defining a transform expression.
</p>
<LearnMore href={`${getLink('DOCS')}/ingestion/index.html#timestampspec`} />
</Callout>
);
});
export const TransformMessage = React.memo(function TransformMessage() {
return (
<Callout className="intro">
<p>
Druid can perform per-row{' '}
<ExternalLink href={`${getLink('DOCS')}/ingestion/transform-spec.html#transforms`}>
transforms
</ExternalLink>{' '}
of column values allowing you to create new derived columns or alter existing column.
</p>
<LearnMore href={`${getLink('DOCS')}/ingestion/index.html#transforms`} />
</Callout>
);
});
export const FilterMessage = React.memo(function FilterMessage() {
return (
<Callout className="intro">
<p>
Druid can filter out unwanted data by applying per-row{' '}
<ExternalLink href={`${getLink('DOCS')}/querying/filters.html`}>filters</ExternalLink>.
</p>
<LearnMore href={`${getLink('DOCS')}/ingestion/index.html#filter`} />
</Callout>
);
});
export interface SchemaMessageProps {
dimensionMode: DimensionMode;
}
export const SchemaMessage = React.memo(function SchemaMessage(props: SchemaMessageProps) {
const { dimensionMode } = props;
return (
<Callout className="intro">
<p>
Each column in Druid must have an assigned type (string, long, float, double, complex, etc).
</p>
{dimensionMode === 'specific' && (
<p>
Default primitive types have been automatically assigned to your columns. If you want to
change the type, click on the column header.
</p>
)}
<LearnMore href={`${getLink('DOCS')}/ingestion/schema-design.html`} />
</Callout>
);
});
export const PartitionMessage = React.memo(function PartitionMessage() {
return (
<Callout className="intro">
<p>Configure how Druid will partition data.</p>
<LearnMore href={`${getLink('DOCS')}/ingestion/index.html#partitioning`} />
</Callout>
);
});
export const TuningMessage = React.memo(function TuningMessage() {
return (
<Callout className="intro">
<p>Fine tune how Druid will ingest data.</p>
<LearnMore href={`${getLink('DOCS')}/ingestion/index.html#tuningconfig`} />
</Callout>
);
});
export const PublishMessage = React.memo(function PublishMessage() {
return (
<Callout className="intro">
<p>Configure behavior of indexed data once it reaches Druid.</p>
</Callout>
);
});
export const SpecMessage = React.memo(function SpecMessage() {
return (
<Callout className="intro">
<p>
Druid begins ingesting data once you submit a JSON ingestion spec. If you modify any values
in this view, the values entered in previous sections will update accordingly. If you modify
any values in previous sections, this spec will automatically update.
</p>
<p>Submit the spec to begin loading data into Druid.</p>
<LearnMore href={`${getLink('DOCS')}/ingestion/index.html#ingestion-specs`} />
</Callout>
);
});

1273
web-console/src/views/load-data-view/load-data-view.tsx
View File

File diff suppressed because it is too large Load Diff

66
web-console/src/views/load-data-view/reorder-menu/reorder-menu.tsx Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Menu, MenuItem } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
export interface ReorderMenuProps {
things: any[] | undefined;
selectedIndex: number;
moveTo: (newIndex: number) => void;
}
export const ReorderMenu = React.memo(function ReorderMenu(props: ReorderMenuProps) {
const { things, selectedIndex, moveTo } = props;
if (!Array.isArray(things) || things.length <= 1 || selectedIndex === -1) return null;
const lastIndex = things.length - 1;
return (
<Menu>
{selectedIndex > 0 && (
<MenuItem
icon={IconNames.DOUBLE_CHEVRON_LEFT}
text="Make first"
onClick={() => moveTo(0)}
/>
)}
{selectedIndex > 0 && (
<MenuItem
icon={IconNames.CHEVRON_LEFT}
text="Move left (earlier in the sort order)"
onClick={() => moveTo(selectedIndex - 1)}
/>
)}
{selectedIndex < lastIndex && (
<MenuItem
icon={IconNames.CHEVRON_RIGHT}
text="Move right (later in the sort order)"
onClick={() => moveTo(selectedIndex + 1)}
/>
)}
{selectedIndex < lastIndex && (
<MenuItem
icon={IconNames.DOUBLE_CHEVRON_RIGHT}
text="Make last"
onClick={() => moveTo(lastIndex)}
/>
)}
</Menu>
);
});

104
web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap
View File

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`schema table matches snapshot 1`] = `
exports[`SchemaTable matches snapshot 1`] = `
<div
class="ReactTable schema-table -striped -highlight"
>
@ -19,7 +19,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-th dimension string rt-resizable-header"
role="columnheader"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
tabindex="-1"
>
<div
@ -62,7 +62,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span
class="table-cell plain"
@ -83,7 +83,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -102,7 +102,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -121,7 +121,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -140,7 +140,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -159,7 +159,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -178,7 +178,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -197,7 +197,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -216,7 +216,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -235,7 +235,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -254,7 +254,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -273,7 +273,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -292,7 +292,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -311,7 +311,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -330,7 +330,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -349,7 +349,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -368,7 +368,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -387,7 +387,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -406,7 +406,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -425,7 +425,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -444,7 +444,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -463,7 +463,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -482,7 +482,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -501,7 +501,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -520,7 +520,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -539,7 +539,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -558,7 +558,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -577,7 +577,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -596,7 +596,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -615,7 +615,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -634,7 +634,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -653,7 +653,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -672,7 +672,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -691,7 +691,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -710,7 +710,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -729,7 +729,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -748,7 +748,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -767,7 +767,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -786,7 +786,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -805,7 +805,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -824,7 +824,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -843,7 +843,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -862,7 +862,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -881,7 +881,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -900,7 +900,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -919,7 +919,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -938,7 +938,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -957,7 +957,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -976,7 +976,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 
@ -995,7 +995,7 @@ exports[`schema table matches snapshot 1`] = `
<div
class="rt-td dimension string"
role="gridcell"
style="flex: 100 0 auto; width: 100px;"
style="flex: 100 0 auto; width: 100px; max-width: 100px;"
>
<span>
 

9
web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
View File

@ -21,7 +21,7 @@ import React from 'react';
import { SchemaTable } from './schema-table';
describe('schema table', () => {
describe('SchemaTable', () => {
it('matches snapshot', () => {
const sampleData = {
header: ['c1'],
@ -37,13 +37,16 @@ describe('schema table', () => {
<SchemaTable
sampleBundle={{
headerAndRows: sampleData,
dimensionsSpec: {},
dimensions: [],
metricsSpec: [],
}}
columnFilter=""
selectedAutoDimension={undefined}
selectedDimensionSpecIndex={-1}
selectedMetricSpecIndex={-1}
onDimensionOrMetricSelect={() => {}}
onAutoDimensionSelect={() => {}}
onDimensionSelect={() => {}}
onMetricSelect={() => {}}
/>
);

79
web-console/src/views/load-data-view/schema-table/schema-table.tsx
View File

@ -23,14 +23,13 @@ import ReactTable from 'react-table';
import { TableCell } from '../../../components';
import {
DimensionSpec,
DimensionsSpec,
getDimensionSpecName,
getDimensionSpecType,
getMetricSpecName,
inflateDimensionSpec,
MetricSpec,
} from '../../../druid-models';
import { caseInsensitiveContains, filterMap, sortWithPrefixSuffix } from '../../../utils';
import { caseInsensitiveContains, filterMap } from '../../../utils';
import { HeaderAndRows, SampleEntry } from '../../../utils/sampler';
import './schema-table.scss';
@ -38,15 +37,19 @@ import './schema-table.scss';
export interface SchemaTableProps {
sampleBundle: {
headerAndRows: HeaderAndRows;
dimensionsSpec: DimensionsSpec;
metricsSpec: MetricSpec[];
dimensions: (string | DimensionSpec)[] | undefined;
metricsSpec: MetricSpec[] | undefined;
};
columnFilter: string;
selectedAutoDimension: string | undefined;
selectedDimensionSpecIndex: number;
selectedMetricSpecIndex: number;
onDimensionOrMetricSelect: (
onAutoDimensionSelect: (dimensionName: string) => void;
onDimensionSelect: (
selectedDimensionSpec: DimensionSpec | undefined,
selectedDimensionSpecIndex: number,
) => void;
onMetricSelect: (
selectedMetricSpec: MetricSpec | undefined,
selectedMetricSpecIndex: number,
) => void;
@ -56,28 +59,26 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
const {
sampleBundle,
columnFilter,
selectedAutoDimension,
selectedDimensionSpecIndex,
selectedMetricSpecIndex,
onDimensionOrMetricSelect,
onAutoDimensionSelect,
onDimensionSelect,
onMetricSelect,
} = props;
const { headerAndRows, dimensionsSpec, metricsSpec } = sampleBundle;
const dimensionMetricSortedHeader = sortWithPrefixSuffix(
headerAndRows.header,
['__time'],
metricsSpec.map(getMetricSpecName),
null,
);
const { headerAndRows, dimensions, metricsSpec } = sampleBundle;
return (
<ReactTable
className="schema-table -striped -highlight"
data={headerAndRows.rows}
columns={filterMap(dimensionMetricSortedHeader, (columnName, i) => {
columns={filterMap(headerAndRows.header, (columnName, i) => {
if (!caseInsensitiveContains(columnName, columnFilter)) return;
const metricSpecIndex = metricsSpec.findIndex(m => getMetricSpecName(m) === columnName);
const metricSpec = metricsSpec[metricSpecIndex];
const metricSpecIndex = metricsSpec
? metricsSpec.findIndex(m => getMetricSpecName(m) === columnName)
: -1;
const metricSpec = metricsSpec ? metricsSpec[metricSpecIndex] : undefined;
if (metricSpec) {
const columnClassName = classNames('metric', {
@ -87,9 +88,7 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
Header: (
<div
className="clickable"
onClick={() =>
onDimensionOrMetricSelect(undefined, -1, metricSpec, metricSpecIndex)
}
onClick={() => onMetricSelect(metricSpec, metricSpecIndex)}
>
<div className="column-name">{columnName}</div>
<div className="column-detail">{metricSpec.type}&nbsp;</div>
@ -102,20 +101,20 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
Cell: ({ value }) => <TableCell value={value} />,
};
} else {
const timestamp = columnName === '__time';
const dimensionSpecIndex = dimensionsSpec.dimensions
? dimensionsSpec.dimensions.findIndex(d => getDimensionSpecName(d) === columnName)
const isTimestamp = columnName === '__time';
const dimensionSpecIndex = dimensions
? dimensions.findIndex(d => getDimensionSpecName(d) === columnName)
: -1;
const dimensionSpec = dimensionsSpec.dimensions
? dimensionsSpec.dimensions[dimensionSpecIndex]
: null;
const dimensionSpecType = dimensionSpec ? getDimensionSpecType(dimensionSpec) : null;
const dimensionSpec = dimensions ? dimensions[dimensionSpecIndex] : undefined;
const dimensionSpecType = dimensionSpec ? getDimensionSpecType(dimensionSpec) : undefined;
const columnClassName = classNames(
timestamp ? 'timestamp' : 'dimension',
isTimestamp ? 'timestamp' : 'dimension',
dimensionSpecType || 'string',
{
selected: dimensionSpec && dimensionSpecIndex === selectedDimensionSpecIndex,
selected:
(dimensionSpec && dimensionSpecIndex === selectedDimensionSpecIndex) ||
selectedAutoDimension === columnName,
},
);
return {
@ -123,31 +122,27 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
<div
className="clickable"
onClick={() => {
if (timestamp) {
onDimensionOrMetricSelect(undefined, -1, undefined, -1);
return;
}
if (isTimestamp) return;
if (!dimensionSpec) return;
onDimensionOrMetricSelect(
inflateDimensionSpec(dimensionSpec),
dimensionSpecIndex,
undefined,
-1,
);
if (dimensionSpec) {
onDimensionSelect(inflateDimensionSpec(dimensionSpec), dimensionSpecIndex);
} else {
onAutoDimensionSelect(columnName);
}
}}
>
<div className="column-name">{columnName}</div>
<div className="column-detail">
{timestamp ? 'long (time column)' : dimensionSpecType || 'string (auto)'}&nbsp;
{isTimestamp ? 'long (time column)' : dimensionSpecType || 'string (auto)'}&nbsp;
</div>
</div>
),
headerClassName: columnClassName,
className: columnClassName,
id: String(i),
width: isTimestamp ? 200 : 100,
accessor: (row: SampleEntry) => (row.parsed ? row.parsed[columnName] : null),
Cell: row => <TableCell value={timestamp ? new Date(row.value) : row.value} />,
Cell: row => <TableCell value={isTimestamp ? new Date(row.value) : row.value} />,
};
}
})}

5
web-console/src/views/query-view/query-view.tsx
View File

@ -459,6 +459,7 @@ export class QueryView extends React.PureComponent<QueryViewProps, QueryViewStat
}
}
const someQueryResult = queryResultState.getSomeData();
const runeMode = QueryView.isJsonLike(queryString);
return (
<SplitterLayout
@ -501,10 +502,10 @@ export class QueryView extends React.PureComponent<QueryViewProps, QueryViewStat
</div>
</div>
<div className="output-pane">
{queryResult && (
{someQueryResult && (
<QueryOutput
runeMode={runeMode}
queryResult={queryResult}
queryResult={someQueryResult}
onQueryChange={this.handleQueryChange}
/>
)}