Adds a text box for delta filter that can accept an optional json
object.
This commit is contained in:
Abhishek Radhakrishnan 2024-05-02 15:50:17 -07:00 committed by GitHub
parent 39ada8b9ad
commit 3717554e16
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 44 additions and 1 deletions

View File

@ -1030,6 +1030,27 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
type: 'string', type: 'string',
placeholder: '/path/to/deltaTable', placeholder: '/path/to/deltaTable',
required: true, required: true,
info: (
<>
<p>A full path to the Delta Lake table.</p>
</>
),
},
{
name: 'inputSource.filter',
label: 'Delta filter',
type: 'json',
defaultValue: {},
info: (
<>
<ExternalLink
href={`${getLink('DOCS')}/ingestion/input-sources/#delta-filter-object`}
>
filter
</ExternalLink>
<p>A Delta filter json object to filter Delta Lake scan files.</p>
</>
),
}, },
]; ];

View File

@ -117,7 +117,8 @@ export type InputSourceDesc =
} }
| { | {
type: 'delta'; type: 'delta';
tablePath?: string; tablePath: string;
filter?: string;
} }
| { | {
type: 'sql'; type: 'sql';
@ -624,6 +625,27 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
placeholder: '/path/to/deltaTable', placeholder: '/path/to/deltaTable',
defined: typeIsKnown(KNOWN_TYPES, 'delta'), defined: typeIsKnown(KNOWN_TYPES, 'delta'),
required: true, required: true,
info: (
<>
<p>A full path to the Delta Lake table.</p>
</>
),
},
{
name: 'filter',
label: 'Delta filter',
type: 'json',
placeholder: '{"type": "=", "column": "name", "value": "foo"}',
defined: inputSource => inputSource.type === 'delta' && deepGet(inputSource, 'filter'),
required: false,
info: (
<>
<ExternalLink href={`${getLink('DOCS')}/ingestion/input-sources/#delta-filter-object`}>
filter
</ExternalLink>
<p>A Delta filter json object to filter Delta Lake scan files.</p>
</>
),
}, },
// sql // sql