Web console: Do not put __time in the dimensions list (#11085)

* Do not make time dimensions

* update e2e test

web-console/README.md

@@ -85,6 +85,8 @@ when the test fails. For example, if `e2e-tests/tutorial-batch.spec.ts` fails, i
 Disabling headless mode while running the tests can be helpful. This can be done via the `DRUID_E2E_TEST_HEADLESS`
 environment variable, which defaults to `true`.
 
+Like so: `DRUID_E2E_TEST_HEADLESS=false npm run test-e2e`
+
 #### Running against alternate web console
 
 The environment variable `DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT` can be used to target a web console running on a

web-console/e2e-tests/component/load-data/config/configure-timestamp.ts

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Data loader configure timestamp step configuration.
+ */
+export class ConfigureTimestampConfig {
+  constructor(props: ConfigureTimestampConfigProps) {
+    Object.assign(this, props);
+  }
+}
+
+interface ConfigureTimestampConfigProps {
+  readonly timestampExpression: string;
+}
+
+export interface ConfigureTimestampConfig extends ConfigureTimestampConfigProps {}

web-console/e2e-tests/component/load-data/data-loader.ts

@@ -21,6 +21,7 @@ import * as playwright from 'playwright-chromium';
 import { clickButton, setLabeledInput, setLabeledTextarea } from '../../util/playwright';
 
 import { ConfigureSchemaConfig } from './config/configure-schema';
+import { ConfigureTimestampConfig } from './config/configure-timestamp';
 import { PartitionConfig } from './config/partition';
 import { PublishConfig } from './config/publish';
 import { DataConnector } from './data-connector/data-connector';
@@ -45,7 +46,7 @@ export class DataLoader {
     await this.connect(this.connector, this.connectValidator);
     if (this.connector.needParse) {
       await this.parseData();
-      await this.parseTime();
+      await this.parseTime(this.configureTimestampConfig);
     }
     await this.transform();
     await this.filter();
@@ -81,8 +82,11 @@ export class DataLoader {
     await clickButton(this.page, 'Next: Parse time');
   }
 
-  private async parseTime() {
+  private async parseTime(configureTimestampConfig?: ConfigureTimestampConfig) {
     await this.page.waitForSelector('.parse-time-table');
+    if (configureTimestampConfig) {
+      await this.applyConfigureTimestampConfig(configureTimestampConfig);
+    }
     await clickButton(this.page, 'Next: Transform');
   }
 
@@ -102,6 +106,12 @@ export class DataLoader {
     await clickButton(this.page, 'Next: Partition');
   }
 
+  private async applyConfigureTimestampConfig(configureTimestampConfig: ConfigureTimestampConfig) {
+    await clickButton(this.page, 'Expression');
+    await setLabeledInput(this.page, 'Expression', configureTimestampConfig.timestampExpression);
+    await clickButton(this.page, 'Apply');
+  }
+
   private async applyConfigureSchemaConfig(configureSchemaConfig: ConfigureSchemaConfig) {
     const rollupSelector = '//*[text()="Rollup"]';
     const rollupInput = await this.page.$(`${rollupSelector}/input`);
@@ -161,6 +171,7 @@ interface DataLoaderProps {
   readonly unifiedConsoleUrl: string;
   readonly connector: DataConnector;
   readonly connectValidator: (preview: string) => void;
+  readonly configureTimestampConfig?: ConfigureTimestampConfig;
   readonly configureSchemaConfig: ConfigureSchemaConfig;
   readonly partitionConfig: PartitionConfig;
   readonly publishConfig: PublishConfig;

web-console/e2e-tests/tutorial-batch.spec.ts

@@ -22,17 +22,15 @@ import * as playwright from 'playwright-chromium';
 import { DatasourcesOverview } from './component/datasources/overview';
 import { IngestionOverview } from './component/ingestion/overview';
 import { ConfigureSchemaConfig } from './component/load-data/config/configure-schema';
-import { SegmentGranularity } from './component/load-data/config/partition';
-import { PartitionConfig } from './component/load-data/config/partition';
+import { ConfigureTimestampConfig } from './component/load-data/config/configure-timestamp';
+import { PartitionConfig, SegmentGranularity } from './component/load-data/config/partition';
 import { PublishConfig } from './component/load-data/config/publish';
 import { LocalFileDataConnector } from './component/load-data/data-connector/local-file';
 import { DataLoader } from './component/load-data/data-loader';
 import { QueryOverview } from './component/query/overview';
 import { saveScreenshotIfError } from './util/debug';
-import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid';
-import { UNIFIED_CONSOLE_URL } from './util/druid';
-import { createBrowser } from './util/playwright';
-import { createPage } from './util/playwright';
+import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, UNIFIED_CONSOLE_URL } from './util/druid';
+import { createBrowser, createPage } from './util/playwright';
 import { retryIfJestAssertionError } from './util/retry';
 import { waitTillWebConsoleReady } from './util/setup';
 
@@ -60,26 +58,24 @@ describe('Tutorial: Loading a file', () => {
   it('Loads data from local disk', async () => {
     const testName = 'load-data-from-local-disk-';
     const datasourceName = testName + ALL_SORTS_OF_CHARS + new Date().toISOString();
-    const dataConnector = new LocalFileDataConnector(page, {
-      baseDirectory: DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
-      fileFilter: 'wikiticker-2015-09-12-sampled.json.gz',
-    });
-    const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false });
-    const partitionConfig = new PartitionConfig({
-      segmentGranularity: SegmentGranularity.DAY,
-      timeIntervals: null,
-      partitionsSpec: null,
-    });
-    const publishConfig = new PublishConfig({ datasourceName: datasourceName });
-
     const dataLoader = new DataLoader({
       page: page,
       unifiedConsoleUrl: UNIFIED_CONSOLE_URL,
-      connector: dataConnector,
+      connector: new LocalFileDataConnector(page, {
+        baseDirectory: DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
+        fileFilter: 'wikiticker-2015-09-12-sampled.json.gz',
+      }),
       connectValidator: validateConnectLocalData,
-      configureSchemaConfig: configureSchemaConfig,
-      partitionConfig: partitionConfig,
-      publishConfig: publishConfig,
+      configureTimestampConfig: new ConfigureTimestampConfig({
+        timestampExpression: 'timestamp_parse("time") + 1',
+      }),
+      configureSchemaConfig: new ConfigureSchemaConfig({ rollup: false }),
+      partitionConfig: new PartitionConfig({
+        segmentGranularity: SegmentGranularity.DAY,
+        timeIntervals: null,
+        partitionsSpec: null,
+      }),
+      publishConfig: new PublishConfig({ datasourceName: datasourceName }),
     });
 
     await saveScreenshotIfError(testName, page, async () => {
@@ -176,7 +172,7 @@ async function validateQuery(page: playwright.Page, datasourceName: string) {
   expect(results).toBeDefined();
   expect(results.length).toBeGreaterThan(0);
   expect(results[0]).toStrictEqual([
-    /* __time */ '2015-09-12T00:46:58.771Z',
+    /* __time */ '2015-09-12T00:46:58.772Z',
     /* added */ '36',
     /* channel */ '#en.wikipedia',
     /* cityName */ 'null',
@@ -195,6 +191,7 @@ async function validateQuery(page: playwright.Page, datasourceName: string) {
     /* page */ 'Talk:Oswald Tilghman',
     /* regionIsoCode */ 'null',
     /* regionName */ 'null',
+    /* time */ '2015-09-12T00:46:58.771Z',
     /* user */ 'GELongstreet',
   ]);
 }

web-console/src/druid-models/transform-spec.spec.ts

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getDimensionNamesFromTransforms } from './transform-spec';
+
+describe('transform-spec', () => {
+  describe('getDimensionNamesFromTransforms', () => {
+    it('does not return the __time column', () => {
+      expect(
+        getDimensionNamesFromTransforms([
+          {
+            type: 'expression',
+            name: 'fooPage',
+            expression: "concat('foo' + page)",
+          },
+          {
+            name: '__time',
+            type: 'expression',
+            expression: "timestamp_shift(timestamp_parse(timestamp), 'P3Y', 1)",
+          },
+          {
+            type: 'expression',
+            name: 'barPage',
+            expression: "concat('foo' + page)",
+          },
+        ]),
+      ).toEqual(['fooPage', 'barPage']);
+    });
+  });
+});

web-console/src/druid-models/transform-spec.tsx

@@ -22,6 +22,8 @@ import React from 'react';
 import { ExternalLink, Field } from '../components';
 import { getLink } from '../links';
 
+import { TIME_COLUMN } from './timestamp-spec';
+
 export interface TransformSpec {
   transforms?: Transform[];
   filter?: Record<string, any>;
@@ -92,7 +94,7 @@ export function getTimestampExpressionFields(transforms: Transform[]): Field<Tra
 export function addTimestampTransform(transforms: Transform[]): Transform[] {
   return [
     {
-      name: '__time',
+      name: TIME_COLUMN,
       type: 'expression',
       expression: '',
     },
@@ -100,6 +102,10 @@ export function addTimestampTransform(transforms: Transform[]): Transform[] {
 }
 
 export function removeTimestampTransform(transforms: Transform[]): Transform[] | undefined {
-  const newTransforms = transforms.filter(transform => transform.name !== '__time');
+  const newTransforms = transforms.filter(transform => transform.name !== TIME_COLUMN);
   return newTransforms.length ? newTransforms : undefined;
 }
+
+export function getDimensionNamesFromTransforms(transforms: Transform[]): string[] {
+  return transforms.map(t => t.name).filter(n => n !== TIME_COLUMN);
+}

web-console/src/utils/sampler.ts

@@ -20,6 +20,7 @@ import * as JSONBig from 'json-bigint-native';
 
 import {
   DimensionsSpec,
+  getDimensionNamesFromTransforms,
   getSpecType,
   getTimestampSchema,
   IngestionSpec,
@@ -464,7 +465,7 @@ export async function sampleForTransform(
         sampleResponse: sampleResponseHack,
         ignoreTimeColumn: true,
         columnOrder: [TIME_COLUMN].concat(inputFormatColumns),
-      }).concat(transforms.map(t => t.name)),
+      }).concat(getDimensionNamesFromTransforms(transforms)),
     );
   }
 
@@ -523,7 +524,7 @@ export async function sampleForFilter(
         sampleResponse: sampleResponseHack,
         ignoreTimeColumn: true,
         columnOrder: [TIME_COLUMN].concat(inputFormatColumns),
-      }).concat(transforms.map(t => t.name)),
+      }).concat(getDimensionNamesFromTransforms(transforms)),
     );
   }