Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update console to core Druid changes (#15854)

This commit is contained in:
Vadim Ogievetsky 2024-02-07 06:14:25 -08:00 committed by GitHub
parent 23d4fade90
commit f2b242b6e6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 185 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/data-management/automatic-compaction.md
View File

@ -157,7 +157,7 @@ You can use concurrent append and replace to safely replace the existing data in
To do this, you need to update your datasource to allow concurrent append and replace tasks:
* If you're using the API, include the following `taskContext` property in your API call: `"useConcurrentLocks": true`
* If you're using the UI, enable **Allow concurrent compactions (experimental)** in the **Compaction config** for your datasource.
* If you're using the UI, enable **Use concurrent locks (experimental)** in the **Compaction config** for your datasource.
You'll also need to update your ingestion jobs for the datasource to include the task context `"useConcurrentLocks": true`.

4
docs/ingestion/concurrent-append-replace.md
View File

@ -36,7 +36,7 @@ If you want to append data to a datasource while compaction is running, you need
### Update the compaction settings with the UI
In the **Compaction config** for a datasource, enable **Allow concurrent compactions (experimental)**.
In the **Compaction config** for a datasource, enable **Use concurrent locks (experimental)**.
For details on accessing the compaction config in the UI, see [Enable automatic compaction with the web console](../data-management/automatic-compaction.md#web-console).
@ -63,7 +63,7 @@ You can provide the context parameter like any other parameter for ingestion job
### Add a task lock using the Druid console
As part of the **Load data** wizard for classic batch (JSON-based ingestion) and streaming ingestion, enable the following config on the **Publish** step: **Allow concurrent tasks (experimental)**.
As part of the **Load data** wizard for classic batch (JSON-based ingestion) and streaming ingestion, enable the following config on the **Publish** step: **Use concurrent locks (experimental)**.
### Add the task lock through the API

2
web-console/lib/keywords.js
View File

@ -68,6 +68,8 @@ exports.SQL_KEYWORDS = [
'PRECEDING',
'FOLLOWING',
'EXTEND',
'PIVOT',
'UNPIVOT',
];
exports.SQL_EXPRESSION_PARTS = [

92
web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap
View File

@ -350,11 +350,32 @@ exports[`CompactionConfigDialog matches snapshot with compactionConfig (dynamic
}
onChange={[Function]}
/>
<Memo(FormGroupWithInfo)
info={
<Memo(PopoverText)>
<p>
If you want to append data to a datasource while compaction is running, you need to enable concurrent append and replace for the datasource by updating the compaction settings.
</p>
<p>
For more information refer to the
<Memo(ExternalLink)
href="https://druid.apache.org/docs/latest/ingestion/concurrent-append-replace.html"
>
documentation
</Memo(ExternalLink)>
.
</p>
</Memo(PopoverText)>
}
inlineInfo={true}
>
<Blueprint4.Switch
checked={false}
label="Allow concurrent compactions (experimental)"
label="Use concurrent locks (experimental)"
onChange={[Function]}
/>
</Memo(FormGroupWithInfo)>
</React.Fragment>
</div>
<div
@ -739,11 +760,32 @@ exports[`CompactionConfigDialog matches snapshot with compactionConfig (hashed p
}
onChange={[Function]}
/>
<Memo(FormGroupWithInfo)
info={
<Memo(PopoverText)>
<p>
If you want to append data to a datasource while compaction is running, you need to enable concurrent append and replace for the datasource by updating the compaction settings.
</p>
<p>
For more information refer to the
<Memo(ExternalLink)
href="https://druid.apache.org/docs/latest/ingestion/concurrent-append-replace.html"
>
documentation
</Memo(ExternalLink)>
.
</p>
</Memo(PopoverText)>
}
inlineInfo={true}
>
<Blueprint4.Switch
checked={false}
label="Allow concurrent compactions (experimental)"
label="Use concurrent locks (experimental)"
onChange={[Function]}
/>
</Memo(FormGroupWithInfo)>
</React.Fragment>
</div>
<div
@ -1128,11 +1170,32 @@ exports[`CompactionConfigDialog matches snapshot with compactionConfig (range pa
}
onChange={[Function]}
/>
<Memo(FormGroupWithInfo)
info={
<Memo(PopoverText)>
<p>
If you want to append data to a datasource while compaction is running, you need to enable concurrent append and replace for the datasource by updating the compaction settings.
</p>
<p>
For more information refer to the
<Memo(ExternalLink)
href="https://druid.apache.org/docs/latest/ingestion/concurrent-append-replace.html"
>
documentation
</Memo(ExternalLink)>
.
</p>
</Memo(PopoverText)>
}
inlineInfo={true}
>
<Blueprint4.Switch
checked={false}
label="Allow concurrent compactions (experimental)"
label="Use concurrent locks (experimental)"
onChange={[Function]}
/>
</Memo(FormGroupWithInfo)>
</React.Fragment>
</div>
<div
@ -1517,11 +1580,32 @@ exports[`CompactionConfigDialog matches snapshot without compactionConfig 1`] =
}
onChange={[Function]}
/>
<Memo(FormGroupWithInfo)
info={
<Memo(PopoverText)>
<p>
If you want to append data to a datasource while compaction is running, you need to enable concurrent append and replace for the datasource by updating the compaction settings.
</p>
<p>
For more information refer to the
<Memo(ExternalLink)
href="https://druid.apache.org/docs/latest/ingestion/concurrent-append-replace.html"
>
documentation
</Memo(ExternalLink)>
.
</p>
</Memo(PopoverText)>
}
inlineInfo={true}
>
<Blueprint4.Switch
checked={false}
label="Allow concurrent compactions (experimental)"
label="Use concurrent locks (experimental)"
onChange={[Function]}
/>
</Memo(FormGroupWithInfo)>
</React.Fragment>
</div>
<div

42
web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx
View File

@ -20,12 +20,20 @@ import { Button, Callout, Classes, Code, Dialog, Intent, Switch } from '@bluepri
import React, { useState } from 'react';
import type { FormJsonTabs } from '../../components';
import { AutoForm, FormJsonSelector, JsonInput } from '../../components';
import {
AutoForm,
ExternalLink,
FormGroupWithInfo,
FormJsonSelector,
JsonInput,
PopoverText,
} from '../../components';
import type { CompactionConfig } from '../../druid-models';
import {
COMPACTION_CONFIG_FIELDS,
compactionConfigHasLegacyInputSegmentSizeBytesSet,
} from '../../druid-models';
import { getLink } from '../../links';
import { deepDelete, deepGet, deepSet, formatBytesCompact } from '../../utils';
import { CompactionHistoryDialog } from '../compaction-history-dialog/compaction-history-dialog';
@ -102,17 +110,39 @@ export const CompactionConfigDialog = React.memo(function CompactionConfigDialog
model={currentConfig}
onChange={m => setCurrentConfig(m as CompactionConfig)}
/>
<FormGroupWithInfo
inlineInfo
info={
<PopoverText>
<p>
If you want to append data to a datasource while compaction is running, you need
to enable concurrent append and replace for the datasource by updating the
compaction settings.
</p>
<p>
For more information refer to the{' '}
<ExternalLink
href={`${getLink('DOCS')}/ingestion/concurrent-append-replace.html`}
>
documentation
</ExternalLink>
.
</p>
</PopoverText>
}
>
<Switch
label="Allow concurrent compactions (experimental)"
checked={typeof deepGet(currentConfig, 'taskContext.taskLockType') === 'string'}
label="Use concurrent locks (experimental)"
checked={Boolean(deepGet(currentConfig, 'taskContext.useConcurrentLocks'))}
onChange={() => {
setCurrentConfig(
(typeof deepGet(currentConfig, 'taskContext.taskLockType') === 'string'
? deepDelete(currentConfig, 'taskContext.taskLockType')
: deepSet(currentConfig, 'taskContext.taskLockType', 'REPLACE')) as any,
(deepGet(currentConfig, 'taskContext.useConcurrentLocks')
? deepDelete(currentConfig, 'taskContext.useConcurrentLocks')
: deepSet(currentConfig, 'taskContext.useConcurrentLocks', true)) as any,
);
}}
/>
</FormGroupWithInfo>
</>
) : (
<JsonInput

9
web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
View File

@ -72,7 +72,7 @@ const CURRENT_YEAR = new Date().getUTCFullYear();
export interface IngestionSpec {
readonly type: IngestionType;
readonly spec: IngestionSpecInner;
readonly context?: { taskLockType?: 'APPEND' | 'REPLACE' };
readonly context?: { useConcurrentLocks?: boolean };
readonly suspended?: boolean;
}
@ -414,13 +414,6 @@ export function normalizeSpec(spec: Partial<IngestionSpec>): IngestionSpec {
spec = deepSetIfUnset(spec, 'spec.ioConfig.type', specType);
spec = deepSetIfUnset(spec, 'spec.tuningConfig.type', specType);
if (spec.context?.taskLockType !== undefined) {
spec.context.taskLockType =
isStreamingSpec(spec) || deepGet(spec, 'spec.ioConfig.appendToExisting')
? 'APPEND'
: 'REPLACE';
}
return spec as IngestionSpec;
}

38
web-console/src/views/load-data-view/load-data-view.tsx
View File

@ -3236,23 +3236,39 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
model={spec}
onChange={this.updateSpec}
/>
<FormGroupWithInfo
inlineInfo
info={
<PopoverText>
<p>
If you want to append data to a datasource while compaction is running, you need
to enable concurrent append and replace for the datasource by updating the
compaction settings.
</p>
<p>
For more information refer to the{' '}
<ExternalLink
href={`${getLink('DOCS')}/ingestion/concurrent-append-replace.html`}
>
documentation
</ExternalLink>
.
</p>
</PopoverText>
}
>
<Switch
label="Allow concurrent tasks (experimental)"
checked={typeof deepGet(spec, 'context.taskLockType') === 'string'}
label="Use concurrent locks (experimental)"
checked={Boolean(deepGet(spec, 'context.useConcurrentLocks'))}
onChange={() => {
this.updateSpec(
typeof deepGet(spec, 'context.taskLockType') === 'string'
? deepDelete(spec, 'context.taskLockType')
: deepSet(
spec,
'context.taskLockType',
isStreamingSpec(spec) || deepGet(spec, 'spec.ioConfig.appendToExisting')
? 'APPEND'
: 'REPLACE',
),
deepGet(spec, 'context.useConcurrentLocks')
? deepDelete(spec, 'context.useConcurrentLocks')
: deepSet(spec, 'context.useConcurrentLocks', true),
);
}}
/>
</FormGroupWithInfo>
</div>
<div className="other">
<H5>Parse error reporting</H5>