From 8432d82c48bf971fb9be1ffd777cc987e1d88b11 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Mon, 12 Apr 2021 09:48:10 -0700 Subject: [PATCH] Web console: Do not put __time in the dimensions list (#11085) * Do not make time dimensions * update e2e test --- web-console/README.md | 2 + .../load-data/config/configure-timestamp.ts | 32 +++++++++++++ .../component/load-data/data-loader.ts | 15 ++++++- web-console/e2e-tests/tutorial-batch.spec.ts | 43 +++++++++--------- .../src/druid-models/transform-spec.spec.ts | 45 +++++++++++++++++++ .../src/druid-models/transform-spec.tsx | 10 ++++- web-console/src/utils/sampler.ts | 5 ++- 7 files changed, 123 insertions(+), 29 deletions(-) create mode 100644 web-console/e2e-tests/component/load-data/config/configure-timestamp.ts create mode 100644 web-console/src/druid-models/transform-spec.spec.ts diff --git a/web-console/README.md b/web-console/README.md index cd5737070d0..8b8618eedf8 100644 --- a/web-console/README.md +++ b/web-console/README.md @@ -85,6 +85,8 @@ when the test fails. For example, if `e2e-tests/tutorial-batch.spec.ts` fails, i Disabling headless mode while running the tests can be helpful. This can be done via the `DRUID_E2E_TEST_HEADLESS` environment variable, which defaults to `true`. +Like so: `DRUID_E2E_TEST_HEADLESS=false npm run test-e2e` + #### Running against alternate web console The environment variable `DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT` can be used to target a web console running on a diff --git a/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts b/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts new file mode 100644 index 00000000000..73e9bfdff30 --- /dev/null +++ b/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Data loader configure timestamp step configuration. + */ +export class ConfigureTimestampConfig { + constructor(props: ConfigureTimestampConfigProps) { + Object.assign(this, props); + } +} + +interface ConfigureTimestampConfigProps { + readonly timestampExpression: string; +} + +export interface ConfigureTimestampConfig extends ConfigureTimestampConfigProps {} diff --git a/web-console/e2e-tests/component/load-data/data-loader.ts b/web-console/e2e-tests/component/load-data/data-loader.ts index df16e715875..b71555e373c 100644 --- a/web-console/e2e-tests/component/load-data/data-loader.ts +++ b/web-console/e2e-tests/component/load-data/data-loader.ts @@ -21,6 +21,7 @@ import * as playwright from 'playwright-chromium'; import { clickButton, setLabeledInput, setLabeledTextarea } from '../../util/playwright'; import { ConfigureSchemaConfig } from './config/configure-schema'; +import { ConfigureTimestampConfig } from './config/configure-timestamp'; import { PartitionConfig } from './config/partition'; import { PublishConfig } from './config/publish'; import { DataConnector } from './data-connector/data-connector'; @@ -45,7 +46,7 @@ export class DataLoader { await this.connect(this.connector, this.connectValidator); if (this.connector.needParse) { await this.parseData(); - await this.parseTime(); + await this.parseTime(this.configureTimestampConfig); } await this.transform(); await this.filter(); @@ -81,8 +82,11 @@ export class DataLoader { await clickButton(this.page, 'Next: Parse time'); } - private async parseTime() { + private async parseTime(configureTimestampConfig?: ConfigureTimestampConfig) { await this.page.waitForSelector('.parse-time-table'); + if (configureTimestampConfig) { + await this.applyConfigureTimestampConfig(configureTimestampConfig); + } await clickButton(this.page, 'Next: Transform'); } @@ -102,6 +106,12 @@ export class DataLoader { await clickButton(this.page, 'Next: Partition'); } + private async applyConfigureTimestampConfig(configureTimestampConfig: ConfigureTimestampConfig) { + await clickButton(this.page, 'Expression'); + await setLabeledInput(this.page, 'Expression', configureTimestampConfig.timestampExpression); + await clickButton(this.page, 'Apply'); + } + private async applyConfigureSchemaConfig(configureSchemaConfig: ConfigureSchemaConfig) { const rollupSelector = '//*[text()="Rollup"]'; const rollupInput = await this.page.$(`${rollupSelector}/input`); @@ -161,6 +171,7 @@ interface DataLoaderProps { readonly unifiedConsoleUrl: string; readonly connector: DataConnector; readonly connectValidator: (preview: string) => void; + readonly configureTimestampConfig?: ConfigureTimestampConfig; readonly configureSchemaConfig: ConfigureSchemaConfig; readonly partitionConfig: PartitionConfig; readonly publishConfig: PublishConfig; diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts index 842d7a50182..ebb95448cab 100644 --- a/web-console/e2e-tests/tutorial-batch.spec.ts +++ b/web-console/e2e-tests/tutorial-batch.spec.ts @@ -22,17 +22,15 @@ import * as playwright from 'playwright-chromium'; import { DatasourcesOverview } from './component/datasources/overview'; import { IngestionOverview } from './component/ingestion/overview'; import { ConfigureSchemaConfig } from './component/load-data/config/configure-schema'; -import { SegmentGranularity } from './component/load-data/config/partition'; -import { PartitionConfig } from './component/load-data/config/partition'; +import { ConfigureTimestampConfig } from './component/load-data/config/configure-timestamp'; +import { PartitionConfig, SegmentGranularity } from './component/load-data/config/partition'; import { PublishConfig } from './component/load-data/config/publish'; import { LocalFileDataConnector } from './component/load-data/data-connector/local-file'; import { DataLoader } from './component/load-data/data-loader'; import { QueryOverview } from './component/query/overview'; import { saveScreenshotIfError } from './util/debug'; -import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid'; -import { UNIFIED_CONSOLE_URL } from './util/druid'; -import { createBrowser } from './util/playwright'; -import { createPage } from './util/playwright'; +import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, UNIFIED_CONSOLE_URL } from './util/druid'; +import { createBrowser, createPage } from './util/playwright'; import { retryIfJestAssertionError } from './util/retry'; import { waitTillWebConsoleReady } from './util/setup'; @@ -60,26 +58,24 @@ describe('Tutorial: Loading a file', () => { it('Loads data from local disk', async () => { const testName = 'load-data-from-local-disk-'; const datasourceName = testName + ALL_SORTS_OF_CHARS + new Date().toISOString(); - const dataConnector = new LocalFileDataConnector(page, { - baseDirectory: DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, - fileFilter: 'wikiticker-2015-09-12-sampled.json.gz', - }); - const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false }); - const partitionConfig = new PartitionConfig({ - segmentGranularity: SegmentGranularity.DAY, - timeIntervals: null, - partitionsSpec: null, - }); - const publishConfig = new PublishConfig({ datasourceName: datasourceName }); - const dataLoader = new DataLoader({ page: page, unifiedConsoleUrl: UNIFIED_CONSOLE_URL, - connector: dataConnector, + connector: new LocalFileDataConnector(page, { + baseDirectory: DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, + fileFilter: 'wikiticker-2015-09-12-sampled.json.gz', + }), connectValidator: validateConnectLocalData, - configureSchemaConfig: configureSchemaConfig, - partitionConfig: partitionConfig, - publishConfig: publishConfig, + configureTimestampConfig: new ConfigureTimestampConfig({ + timestampExpression: 'timestamp_parse("time") + 1', + }), + configureSchemaConfig: new ConfigureSchemaConfig({ rollup: false }), + partitionConfig: new PartitionConfig({ + segmentGranularity: SegmentGranularity.DAY, + timeIntervals: null, + partitionsSpec: null, + }), + publishConfig: new PublishConfig({ datasourceName: datasourceName }), }); await saveScreenshotIfError(testName, page, async () => { @@ -176,7 +172,7 @@ async function validateQuery(page: playwright.Page, datasourceName: string) { expect(results).toBeDefined(); expect(results.length).toBeGreaterThan(0); expect(results[0]).toStrictEqual([ - /* __time */ '2015-09-12T00:46:58.771Z', + /* __time */ '2015-09-12T00:46:58.772Z', /* added */ '36', /* channel */ '#en.wikipedia', /* cityName */ 'null', @@ -195,6 +191,7 @@ async function validateQuery(page: playwright.Page, datasourceName: string) { /* page */ 'Talk:Oswald Tilghman', /* regionIsoCode */ 'null', /* regionName */ 'null', + /* time */ '2015-09-12T00:46:58.771Z', /* user */ 'GELongstreet', ]); } diff --git a/web-console/src/druid-models/transform-spec.spec.ts b/web-console/src/druid-models/transform-spec.spec.ts new file mode 100644 index 00000000000..b221e34d4c1 --- /dev/null +++ b/web-console/src/druid-models/transform-spec.spec.ts @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getDimensionNamesFromTransforms } from './transform-spec'; + +describe('transform-spec', () => { + describe('getDimensionNamesFromTransforms', () => { + it('does not return the __time column', () => { + expect( + getDimensionNamesFromTransforms([ + { + type: 'expression', + name: 'fooPage', + expression: "concat('foo' + page)", + }, + { + name: '__time', + type: 'expression', + expression: "timestamp_shift(timestamp_parse(timestamp), 'P3Y', 1)", + }, + { + type: 'expression', + name: 'barPage', + expression: "concat('foo' + page)", + }, + ]), + ).toEqual(['fooPage', 'barPage']); + }); + }); +}); diff --git a/web-console/src/druid-models/transform-spec.tsx b/web-console/src/druid-models/transform-spec.tsx index c1dc3400302..18ecee09d4c 100644 --- a/web-console/src/druid-models/transform-spec.tsx +++ b/web-console/src/druid-models/transform-spec.tsx @@ -22,6 +22,8 @@ import React from 'react'; import { ExternalLink, Field } from '../components'; import { getLink } from '../links'; +import { TIME_COLUMN } from './timestamp-spec'; + export interface TransformSpec { transforms?: Transform[]; filter?: Record; @@ -92,7 +94,7 @@ export function getTimestampExpressionFields(transforms: Transform[]): Field transform.name !== '__time'); + const newTransforms = transforms.filter(transform => transform.name !== TIME_COLUMN); return newTransforms.length ? newTransforms : undefined; } + +export function getDimensionNamesFromTransforms(transforms: Transform[]): string[] { + return transforms.map(t => t.name).filter(n => n !== TIME_COLUMN); +} diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts index 0bd9ae8e36c..a18c66faffa 100644 --- a/web-console/src/utils/sampler.ts +++ b/web-console/src/utils/sampler.ts @@ -20,6 +20,7 @@ import * as JSONBig from 'json-bigint-native'; import { DimensionsSpec, + getDimensionNamesFromTransforms, getSpecType, getTimestampSchema, IngestionSpec, @@ -464,7 +465,7 @@ export async function sampleForTransform( sampleResponse: sampleResponseHack, ignoreTimeColumn: true, columnOrder: [TIME_COLUMN].concat(inputFormatColumns), - }).concat(transforms.map(t => t.name)), + }).concat(getDimensionNamesFromTransforms(transforms)), ); } @@ -523,7 +524,7 @@ export async function sampleForFilter( sampleResponse: sampleResponseHack, ignoreTimeColumn: true, columnOrder: [TIME_COLUMN].concat(inputFormatColumns), - }).concat(transforms.map(t => t.name)), + }).concat(getDimensionNamesFromTransforms(transforms)), ); }