Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console basic end-to-end-test (#9595) 

Load data and query (i.e., automate
https://druid.apache.org/docs/latest/tutorials/tutorial-batch.html) to
have some basic checks ensuring the web console is wired up to druid
correctly.

The new end-to-end tests (tutorial-batch.spec.ts) are added to
`web-console/e2e-tests`. Within that directory:
- `components` represent the various tabs of the web console. Currently,
  abstractions for `load data`, `ingestion`, `datasources`, and `query`
  are implemented.
- `components/load-data/data-connector` contains abstractions for the
  different data source options available to the data loader's `Connect`
  step. Currently, only the `Local file` data source connector is
  implemented.
- `components/load-data/config` contains abstractions for the different
  configuration options available for each step of the data loader flow.
  Currently, the `Configure Schema`, `Partition`, and `Publish` steps
  have initial implementation of their configuration options.
- `util` contains various helper methods for the tests and does not
  contain abstractions of the web console.

Changes to add the new tests to CI:
- `.travis.yml`: New "web console end-to-end tests" job
- `web-console/jest.*.js`: Refactor jest configurations to have
  different flavors for unit tests and for end-to-end tests. In
  particular, the latter adds a jest setup configuration to wait for the
  web console to be ready (`web-console/e2e-tests/util/setup.ts`).
- `web-console/package.json`: Refactor run scripts to add new script for
  running end-to-end tests.
- `web-console/script/druid`: Utility scripts for building, starting,
  and stopping druid.

Other changes:
- `pom.xml`: Refactor various settings disable java static checks and to
  disable java tests into two new maven profiles. Since the same
  settings are used in several places (e.g., .travis.yml, Dockerfiles,
  etc.), having them in maven profiles makes it more maintainable.
- `web-console/src/console-application.tsx`: Fix typo ("the the").
This commit is contained in:
Chi Cao Minh 2020-04-09 12:38:09 -07:00 committed by GitHub
parent ab5ac7f890
commit 84c1c2505d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 1494 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -11,6 +11,7 @@ target
.project
.PVS-Studio
.settings/
.vscode
*.log
*.DS_Store
_site

23
.travis.yml
View File

@ -30,17 +30,8 @@ env:
- DOCKER_IP=127.0.0.1 # for integration tests
- MVN="mvn -B"
- > # Various options to make execution of maven goals faster (e.g., mvn install)
MAVEN_SKIP="
-Danimal.sniffer.skip=true
-Dcheckstyle.skip=true
-Ddruid.console.skip=true
-Denforcer.skip=true
-Dforbiddenapis.skip=true
-Dmaven.javadoc.skip=true
-Dpmd.skip=true
-Dspotbugs.skip=true
"
- MAVEN_SKIP_TESTS="-DskipTests -Djacoco.skip=true"
MAVEN_SKIP="-Pskip-static-checks -Ddruid.console.skip=true -Dmaven.javadoc.skip=true"
- MAVEN_SKIP_TESTS="-Pskip-tests"
# Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not
# needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any
@ -254,14 +245,20 @@ jobs:
name: "(openjdk11) other modules test (SQL Compatibility)"
jdk: openjdk11
- &test_webconsole
name: "web console"
- name: "web console"
install: skip
script:
- ${MVN} test -pl 'web-console'
after_success:
- (cd web-console && travis_retry npm run codecov) # retry in case of network error
- name: "web console end-to-end test"
before_install: *setup_generate_license
install: web-console/script/druid build
before_script: web-console/script/druid start
script: (cd web-console && npm run test-e2e)
after_script: web-console/script/druid stop
- name: "docs"
install: (cd website && npm install)
script: |-

10
distribution/docker/Dockerfile
View File

@ -28,14 +28,8 @@ WORKDIR /src
RUN mvn -B -ff -q dependency:go-offline \
install \
-Pdist,bundle-contrib-exts \
-DskipTests \
-Danimal.sniffer.skip=true \
-Dcheckstyle.skip=true \
-Denforcer.skip=true \
-Dforbiddenapis.skip=true \
-Dmaven.javadoc.skip=true \
-Dpmd.skip=true \
-Dspotbugs.skip=true
-Pskip-static-checks,skip-tests \
-Dmaven.javadoc.skip=true
RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \
-Dexpression=project.version -DforceStdout=true \

10
distribution/docker/Dockerfile.java11
View File

@ -28,14 +28,8 @@ WORKDIR /src
RUN mvn -B -ff -q dependency:go-offline \
install \
-Pdist,bundle-contrib-exts \
-DskipTests \
-Danimal.sniffer.skip=true \
-Dcheckstyle.skip=true \
-Denforcer.skip=true \
-Dforbiddenapis.skip=true \
-Dmaven.javadoc.skip=true \
-Dpmd.skip=true \
-Dspotbugs.skip=true
-Pskip-static-checks,skip-tests \
-Dmaven.javadoc.skip=true
RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \
-Dexpression=project.version -DforceStdout=true \

19
pom.xml
View File

@ -1947,5 +1947,24 @@
<module>website</module>
</modules>
</profile>
<profile>
<id>skip-static-checks</id>
<properties>
<animal.sniffer.skip>true</animal.sniffer.skip>
<checkstyle.skip>true</checkstyle.skip>
<enforcer.skip>true</enforcer.skip>
<forbiddenapis.skip>true</forbiddenapis.skip>
<maven.javadoc.skip>true</maven.javadoc.skip>
<pmd.skip>true</pmd.skip>
<spotbugs.skip>true</spotbugs.skip>
</properties>
</profile>
<profile>
<id>skip-tests</id>
<properties>
<skipTests>true</skipTests>
<jacoco.skip>true</jacoco.skip>
</properties>
</profile>
</profiles>
</project>

34
web-console/e2e-tests/component/datasources/datasource.ts Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Represents datasource row in datasource overview table.
*/
export class Datasource {
constructor(props: DatasourceProps) {
Object.assign(this, props);
}
}
interface DatasourceProps {
readonly name: string;
readonly availability: string;
readonly numRows: number;
}
export interface Datasource extends DatasourceProps {}

71
web-console/e2e-tests/component/datasources/overview.ts Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
import { extractTable } from '../../util/table';
import { Datasource } from './datasource';
/**
* Datasource overview table column identifiers.
*/
enum DatasourceColumn {
NAME = 0,
AVAILABILITY,
SEGMENT_LOAD_DROP,
RETENTION,
REPLICATED_SIZE,
SIZE,
COMPACTION,
AVG_SEGMENT_SIZE,
NUM_ROWS,
}
/**
* Represents datasource overview tab.
*/
export class DatasourcesOverview {
private readonly page: playwright.Page;
private readonly baseUrl: string;
constructor(page: playwright.Page, unifiedConsoleUrl: string) {
this.page = page;
this.baseUrl = unifiedConsoleUrl + '#datasources';
}
async getDatasources(): Promise<Datasource[]> {
await this.page.goto(this.baseUrl);
await this.page.reload({ waitUntil: 'networkidle0' });
const data = await extractTable(this.page, 'div div.rt-tr-group', 'div.rt-td');
return data.map(
row =>
new Datasource({
name: row[DatasourceColumn.NAME],
availability: row[DatasourceColumn.AVAILABILITY],
numRows: DatasourcesOverview.parseNumber(row[DatasourceColumn.NUM_ROWS]),
}),
);
}
private static parseNumber(text: string): number {
return Number(text.replace(/,/g, ''));
}
}

65
web-console/e2e-tests/component/ingestion/overview.ts Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
import { extractTable } from '../../util/table';
import { IngestionTask } from './task';
/**
* Ingestion overview task table column identifiers.
*/
enum TaskColumn {
TASK_ID = 0,
GROUP_ID,
TYPE,
DATASOURCE,
LOCATION,
CREATED_TIME,
STATUS,
DURATION,
}
/**
* Represents ingestion overview tab.
*/
export class IngestionOverview {
private readonly page: playwright.Page;
private readonly baseUrl: string;
constructor(page: playwright.Page, unifiedConsoleUrl: string) {
this.page = page;
this.baseUrl = unifiedConsoleUrl + '#ingestion';
}
async getTasks(): Promise<IngestionTask[]> {
await this.page.goto(this.baseUrl);
await this.page.reload({ waitUntil: 'networkidle0' });
const data = await extractTable(this.page, 'div.bottom-pane div.rt-tr-group', 'div.rt-td');
return data.map(
row =>
new IngestionTask({
datasource: row[TaskColumn.DATASOURCE],
status: row[TaskColumn.STATUS],
}),
);
}
}

33
web-console/e2e-tests/component/ingestion/task.ts Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Represents row in ingestion overview task table.
*/
export class IngestionTask {
constructor(props: IngestionTaskProps) {
Object.assign(this, props);
}
}
interface IngestionTaskProps {
readonly datasource: string;
readonly status: string;
}
export interface IngestionTask extends IngestionTaskProps {}

32
web-console/e2e-tests/component/load-data/config/configure-schema.ts Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Data loader configure schema step configuration.
*/
export class ConfigureSchemaConfig {
constructor(props: ConfigureSchemaConfigProps) {
Object.assign(this, props);
}
}
interface ConfigureSchemaConfigProps {
readonly rollup: boolean;
}
export interface ConfigureSchemaConfig extends ConfigureSchemaConfigProps {}

42
web-console/e2e-tests/component/load-data/config/partition.ts Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Possible values for partition step segment granularity.
*/
export enum SegmentGranularity {
HOUR = 'HOUR',
DAY = 'DAY',
MONTH = 'MONTH',
YEAR = 'YEAR',
}
/**
* Data loader partition step configuration.
*/
export class PartitionConfig {
constructor(props: PartitionConfigProps) {
Object.assign(this, props);
}
}
interface PartitionConfigProps {
readonly segmentGranularity: SegmentGranularity;
}
export interface PartitionConfig extends PartitionConfigProps {}

32
web-console/e2e-tests/component/load-data/config/publish.ts Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Data loader publish step configuration.
*/
export class PublishConfig {
constructor(props: PublishConfigProps) {
Object.assign(this, props);
}
}
interface PublishConfigProps {
readonly datasourceName: string;
}
export interface PublishConfig extends PublishConfigProps {}

25
web-console/e2e-tests/component/load-data/data-connector/data-connector.ts Normal file
View File

@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Connector for data loader input data.
*/
export interface DataConnector {
readonly name: string;
connect(): Promise<void>;
}

54
web-console/e2e-tests/component/load-data/data-connector/local-file.ts Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
import { DataConnector } from './data-connector';
/**
* Local file connector for data loader input data.
*/
export class LocalFileDataConnector implements DataConnector {
readonly name: string;
private page: playwright.Page;
private baseDirectory: string;
private fileFilter: string;
constructor(page: playwright.Page, baseDirectory: string, fileFilter: string) {
this.name = 'Local disk';
this.page = page;
this.baseDirectory = baseDirectory;
this.fileFilter = fileFilter;
}
async connect() {
const baseDirectoryInput = await this.page.$('input[placeholder="/path/to/files/"]');
await this.setInput(baseDirectoryInput!, this.baseDirectory);
const fileFilterInput = await this.page.$('input[value="*"]');
await this.setInput(fileFilterInput!, this.fileFilter);
const applyButton = await this.page.$('"Apply"');
await applyButton!.click();
}
private async setInput(input: playwright.ElementHandle<Element>, value: string) {
await input.fill('');
await input.type(value);
}
}

167
web-console/e2e-tests/component/load-data/data-loader.ts Normal file
View File

@ -0,0 +1,167 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
import { ConfigureSchemaConfig } from './config/configure-schema';
import { PartitionConfig } from './config/partition';
import { PublishConfig } from './config/publish';
import { DataConnector } from './data-connector/data-connector';
/**
* Represents load data tab.
*/
export class DataLoader {
private readonly baseUrl: string;
constructor(props: DataLoaderProps) {
Object.assign(this, props);
this.baseUrl = props.unifiedConsoleUrl! + '#load-data';
}
/**
* Execute each step to load data.
*/
async load() {
await this.page.goto(this.baseUrl);
await this.start();
await this.connect(this.connector, this.connectValidator);
await this.parseData();
await this.parseTime();
await this.transform();
await this.filter();
await this.configureSchema(this.configureSchemaConfig);
await this.partition(this.partitionConfig);
await this.tune();
await this.publish(this.publishConfig);
await this.editSpec();
}
private async start() {
await this.page.click(`"${this.connector.name}"`);
await this.clickButton('Connect data');
}
private async connect(connector: DataConnector, validator: (preview: string) => void) {
await connector.connect();
await this.validateConnect(validator);
await this.clickButton('Next: Parse data');
}
private async validateConnect(validator: (preview: string) => void) {
const previewSelector = '.raw-lines';
await this.page.waitFor(previewSelector);
const preview = await this.page.$eval(previewSelector, el => (el as HTMLTextAreaElement).value);
validator(preview!);
}
private async parseData() {
await this.page.waitFor('.parse-data-table');
await this.clickButton('Next: Parse time');
}
private async parseTime() {
await this.page.waitFor('.parse-time-table');
await this.clickButton('Next: Transform');
}
private async transform() {
await this.page.waitFor('.transform-table');
await this.clickButton('Next: Filter');
}
private async filter() {
await this.page.waitFor('.filter-table');
await this.clickButton('Next: Configure schema');
}
private async configureSchema(configureSchemaConfig: ConfigureSchemaConfig) {
await this.page.waitFor('.schema-table');
await this.applyConfigureSchemaConfig(configureSchemaConfig);
await this.clickButton('Next: Partition');
}
private async applyConfigureSchemaConfig(configureSchemaConfig: ConfigureSchemaConfig) {
const rollup = await this.page.$('//*[text()="Rollup"]/input');
const rollupChecked = await rollup!.evaluate(el => (el as HTMLInputElement).checked);
if (rollupChecked !== configureSchemaConfig.rollup) {
await rollup!.click();
await this.clickButton('Yes');
await this.page.waitFor('.recipe-toaster');
}
}
private async partition(partitionConfig: PartitionConfig) {
await this.page.waitFor('div.load-data-view.partition');
await this.applyPartitionConfig(partitionConfig);
await this.clickButton('Next: Tune');
}
private async applyPartitionConfig(partitionConfig: PartitionConfig) {
const segmentGranularity = await this.page.$(
'//*[text()="Segment granularity"]/following-sibling::div//input',
);
await this.setInput(segmentGranularity!, partitionConfig.segmentGranularity);
}
private async tune() {
await this.page.waitFor('div.load-data-view.tuning');
await this.clickButton('Next: Publish');
}
private async publish(publishConfig: PublishConfig) {
await this.page.waitFor('div.load-data-view.publish');
await this.applyPublishConfig(publishConfig);
await this.clickButton('Edit spec');
}
private async applyPublishConfig(publishConfig: PublishConfig) {
if (publishConfig.datasourceName != null) {
const datasourceName = await this.page.$(
'//*[text()="Datasource name"]/following-sibling::div//input',
);
await this.setInput(datasourceName!, publishConfig.datasourceName);
}
}
private async editSpec() {
await this.page.waitFor('div.load-data-view.spec');
await this.clickButton('Submit');
}
private async clickButton(text: string) {
await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any);
}
private async setInput(input: playwright.ElementHandle<Element>, value: string) {
await input.fill('');
await input.type(value);
}
}
interface DataLoaderProps {
readonly page: playwright.Page;
readonly unifiedConsoleUrl: string;
readonly connector: DataConnector;
readonly connectValidator: (preview: string) => void;
readonly configureSchemaConfig: ConfigureSchemaConfig;
readonly partitionConfig: PartitionConfig;
readonly publishConfig: PublishConfig;
}
export interface DataLoader extends DataLoaderProps {}

55
web-console/e2e-tests/component/query/overview.ts Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
import { extractTable } from '../../util/table';
/**
* Represents query overview tab.
*/
export class QueryOverview {
private readonly page: playwright.Page;
private readonly baseUrl: string;
constructor(page: playwright.Page, unifiedConsoleUrl: string) {
this.page = page;
this.baseUrl = unifiedConsoleUrl + '#query';
}
async runQuery(query: string): Promise<string[][]> {
await this.page.goto(this.baseUrl);
await this.page.reload({ waitUntil: 'networkidle0' });
const input = await this.page.$('div.query-input textarea');
await this.setInput(input!, query);
await this.clickButton('Run');
await this.page.waitFor('div.query-info');
return await extractTable(this.page, 'div.query-output div.rt-tr-group', 'div.rt-td');
}
private async setInput(input: playwright.ElementHandle<Element>, value: string) {
await input.fill('');
await input.type(value);
}
private async clickButton(text: string) {
await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any);
}
}

190
web-console/e2e-tests/tutorial-batch.spec.ts Normal file
View File

@ -0,0 +1,190 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
import { v4 as uuid } from 'uuid';
import { DatasourcesOverview } from './component/datasources/overview';
import { IngestionOverview } from './component/ingestion/overview';
import { ConfigureSchemaConfig } from './component/load-data/config/configure-schema';
import { PartitionConfig } from './component/load-data/config/partition';
import { SegmentGranularity } from './component/load-data/config/partition';
import { PublishConfig } from './component/load-data/config/publish';
import { LocalFileDataConnector } from './component/load-data/data-connector/local-file';
import { DataLoader } from './component/load-data/data-loader';
import { QueryOverview } from './component/query/overview';
import { saveScreenshotIfError } from './util/debug';
import { UNIFIED_CONSOLE_URL } from './util/druid';
import { createBrowserNormal as createBrowser } from './util/playwright';
import { createPage } from './util/playwright';
import { retryIfJestAssertionError } from './util/retry';
jest.setTimeout(5 * 60 * 1000);
describe('Tutorial: Loading a file', () => {
let browser: playwright.Browser;
let page: playwright.Page;
beforeAll(async () => {
browser = await createBrowser();
});
beforeEach(async () => {
page = await createPage(browser);
});
afterAll(async () => {
await browser.close();
});
it('Loads data from local disk', async () => {
const datasourceName = uuid();
const dataConnector = new LocalFileDataConnector(
page,
'quickstart/tutorial/',
'wikiticker-2015-09-12-sampled.json.gz',
);
const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false });
const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY });
const publishConfig = new PublishConfig({ datasourceName: datasourceName });
const dataLoader = new DataLoader({
page: page,
unifiedConsoleUrl: UNIFIED_CONSOLE_URL,
connector: dataConnector,
connectValidator: validateConnectLocalData,
configureSchemaConfig: configureSchemaConfig,
partitionConfig: partitionConfig,
publishConfig: publishConfig,
});
await saveScreenshotIfError('load-data-from-local-disk-', page, async () => {
await dataLoader.load();
await validateTaskStatus(page, datasourceName);
await validateDatasourceStatus(page, datasourceName);
await validateQuery(page, datasourceName);
});
});
});
function validateConnectLocalData(preview: string) {
const lines = preview.split('\n');
expect(lines.length).toBe(500);
const firstLine = lines[0];
expect(firstLine).toBe(
'{' +
'"time":"2015-09-12T00:46:58.771Z"' +
',"channel":"#en.wikipedia"' +
',"cityName":null' +
',"comment":"added project"' +
',"countryIsoCode":null' +
',"countryName":null' +
',"isAnonymous":false' +
',"isMinor":false' +
',"isNew":false' +
',"isRobot":false' +
',"isUnpatrolled":false' +
',"metroCode":null' +
',"namespace":"Talk"' +
',"page":"Talk:Oswald Tilghman"' +
',"regionIsoCode":null' +
',"regionName":null' +
',"user":"GELongstreet"' +
',"delta":36' +
',"added":36' +
',"deleted":0' +
'}',
);
const lastLine = lines[lines.length - 1];
expect(lastLine).toBe(
'{' +
'"time":"2015-09-12T01:11:54.823Z"' +
',"channel":"#en.wikipedia"' +
',"cityName":null' +
',"comment":"/* History */[[WP:AWB/T|Typo fixing]], [[WP:AWB/T|typo(s) fixed]]: nothern → northern using [[Project:AWB|AWB]]"' +
',"countryIsoCode":null' +
',"countryName":null' +
',"isAnonymous":false' +
',"isMinor":true' +
',"isNew":false' +
',"isRobot":false' +
',"isUnpatrolled":false' +
',"metroCode":null' +
',"namespace":"Main"' +
',"page":"Hapoel Katamon Jerusalem F.C."' +
',"regionIsoCode":null' +
',"regionName":null' +
',"user":"The Quixotic Potato"' +
',"delta":1' +
',"added":1' +
',"deleted":0' +
'}',
);
}
async function validateTaskStatus(page: playwright.Page, datasourceName: string) {
const ingestionOverview = new IngestionOverview(page, UNIFIED_CONSOLE_URL);
await retryIfJestAssertionError(async () => {
const tasks = await ingestionOverview.getTasks();
const task = tasks.find(t => t.datasource === datasourceName);
expect(task).toBeDefined();
expect(task!.status).toMatch('SUCCESS');
});
}
async function validateDatasourceStatus(page: playwright.Page, datasourceName: string) {
const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
await retryIfJestAssertionError(async () => {
const datasources = await datasourcesOverview.getDatasources();
const datasource = datasources.find(t => t.name === datasourceName);
expect(datasource).toBeDefined();
expect(datasource!.availability).toMatch('Fully available (1 segment)');
expect(datasource!.numRows).toBe(39244);
});
}
async function validateQuery(page: playwright.Page, datasourceName: string) {
const queryOverview = new QueryOverview(page, UNIFIED_CONSOLE_URL);
const query = `SELECT * FROM "${datasourceName}" ORDER BY __time`;
const results = await queryOverview.runQuery(query);
expect(results).toBeDefined();
expect(results.length).toBeGreaterThan(0);
expect(results[0]).toStrictEqual([
/* __time */ '2015-09-12T00:46:58.771Z',
/* added */ '36',
/* channel */ '#en.wikipedia',
/* cityName */ 'null',
/* comment */ 'added project',
/* countryIsoCode */ 'null',
/* countryName */ 'null',
/* deleted */ '0',
/* delta */ '36',
/* isAnonymous */ 'false',
/* isMinor */ 'false',
/* isNew */ 'false',
/* isRobot */ 'false',
/* isUnpatrolled */ 'false',
/* namespace */ 'Talk',
/* page */ 'Talk:Oswald Tilghman',
/* regionIsoCode */ 'null',
/* regionName */ 'null',
/* user */ 'GELongstreet',
]);
}

32
web-console/e2e-tests/util/debug.ts Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
export async function saveScreenshotIfError(
filenamePrefix: string,
page: playwright.Page,
test: () => Promise<void>,
) {
try {
await test();
} catch (e) {
await page.screenshot({ path: filenamePrefix + 'error-screenshot.png' });
throw e;
}
}

19
web-console/e2e-tests/util/druid.ts Normal file
View File

@ -0,0 +1,19 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const UNIFIED_CONSOLE_URL = 'http://localhost:8888/unified-console.html';

50
web-console/e2e-tests/util/playwright.ts Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
const DEBUG = true;
const WIDTH = 1920;
const HEIGHT = 1080;
const PADDING = 128;
export async function createBrowserNormal(): Promise<playwright.Browser> {
return createBrowserInternal(!DEBUG);
}
export async function createBrowserDebug(): Promise<playwright.Browser> {
return createBrowserInternal(DEBUG);
}
async function createBrowserInternal(debug: boolean): Promise<playwright.Browser> {
const launchOptions: any = {
args: [`--window-size=${WIDTH},${HEIGHT + PADDING}`],
};
if (debug) {
launchOptions.headless = false;
launchOptions.slowMo = 20;
}
return playwright.chromium.launch(launchOptions);
}
export async function createPage(browser: playwright.Browser): Promise<playwright.Page> {
const context = await browser.newContext();
const page = await context.newPage();
await page.setViewportSize({ width: WIDTH, height: HEIGHT });
return page;
}

42
web-console/e2e-tests/util/retry.ts Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
function sleep(msec: number) {
return new Promise(resolve => setTimeout(resolve, msec));
}
export async function retryIfJestAssertionError(
callback: () => Promise<void>,
msec = 1000,
maxTries = 60,
) {
let i = 0;
while (true) {
try {
await callback();
return;
} catch (e) {
i++;
if (e.constructor.name === 'JestAssertionError' && i < maxTries) {
await sleep(msec);
} else {
throw e;
}
}
}
}

33
web-console/e2e-tests/util/setup.ts Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { createBrowserNormal } from './playwright';
import { createPage } from './playwright';
(async () => {
const browser = await createBrowserNormal();
try {
const page = await createPage(browser);
await page.waitFor('//*[contains(text(),"console will not function at the moment")]', {
visibility: 'hidden',
});
} finally {
await browser.close();
}
})();

52
web-console/e2e-tests/util/table.ts Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as playwright from 'playwright-core';
/**
* Extracts an HTML table into a text representation.
* @param page Playwright page from which to extract HTML table
* @param tableSelector Playwright selector for table
* @param rowSelector Playwright selector for table row
*/
export async function extractTable(
page: playwright.Page,
tableSelector: string,
rowSelector: string,
): Promise<string[][]> {
await page.waitFor(tableSelector);
return page.evaluate(
(tableSelector, rowSelector) => {
const BLANK_VALUE = '\xa0';
const data = [];
const rows = document.querySelectorAll(tableSelector);
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const columns = row.querySelectorAll(rowSelector);
const values = Array.from(columns).map(c => (c as HTMLElement).innerText);
if (!values.every(value => value === BLANK_VALUE)) {
data.push(values);
}
}
return data;
},
tableSelector,
rowSelector,
);
}

25
web-console/jest.common.config.js Normal file
View File

@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
module.exports = {
"preset": "ts-jest",
"testEnvironment": "jsdom",
"testMatch": [
"**/?(*.)+(spec).ts?(x)"
],
};

25
web-console/jest.e2e.config.js Normal file
View File

@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const common = require('./jest.common.config');
module.exports = Object.assign(common, {
"setupFilesAfterEnv": [
"<rootDir>e2e-tests/util/setup.ts"
]
});

31
web-console/jest.unit.config.js Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const common = require('./jest.common.config');
module.exports = Object.assign(common, {
"moduleNameMapper": {
"\\.s?css$": "identity-obj-proxy"
},
"snapshotSerializers": [
"enzyme-to-json/serializer"
],
"setupFilesAfterEnv": [
"<rootDir>src/setup-tests.ts"
],
});

202
web-console/package-lock.json generated
View File

@ -1593,6 +1593,12 @@
"integrity": "sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==",
"dev": true
},
"@types/uuid": {
"version": "7.0.2",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-7.0.2.tgz",
"integrity": "sha512-8Ly3zIPTnT0/8RCU6Kg/G3uTICf9sRwYOpUzSIM3503tLIKcnJPRuinHhXngJUy2MntrEf6dlpOHXJju90Qh5w==",
"dev": true
},
"@types/vfile": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/vfile/-/vfile-3.0.2.tgz",
@ -4906,6 +4912,29 @@
}
}
},
"extract-zip": {
"version": "1.6.7",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz",
"integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=",
"dev": true,
"requires": {
"concat-stream": "1.6.2",
"debug": "2.6.9",
"mkdirp": "0.5.1",
"yauzl": "2.4.1"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"dev": true,
"requires": {
"ms": "2.0.0"
}
}
}
},
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
@ -4968,6 +4997,15 @@
"bser": "^2.0.0"
}
},
"fd-slicer": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz",
"integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=",
"dev": true,
"requires": {
"pend": "~1.2.0"
}
},
"figgy-pudding": {
"version": "3.5.1",
"resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.1.tgz",
@ -7789,6 +7827,12 @@
}
}
},
"jpeg-js": {
"version": "0.3.7",
"resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.7.tgz",
"integrity": "sha512-9IXdWudL61npZjvLuVe/ktHiA41iE8qFyLB+4VDTblEsWBzeg8WQTlktdUK4CdncUqtUgUg0bbOmTE2bKBKaBQ==",
"dev": true
},
"js-base64": {
"version": "2.5.2",
"resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.5.2.tgz",
@ -9665,6 +9709,12 @@
"sha.js": "^2.4.8"
}
},
"pend": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
"integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=",
"dev": true
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@ -9716,12 +9766,107 @@
"find-up": "^3.0.0"
}
},
"playwright-chromium": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/playwright-chromium/-/playwright-chromium-0.11.1.tgz",
"integrity": "sha512-7b9fdpMaEBDT/bazkfzHGiKGJ7+PzlVoxwnW94lIxV3uDUSSAHxkFwwD1ks1NZnvBQuTE9IksOP9hKYA1gby5A==",
"dev": true,
"requires": {
"playwright-core": "=0.11.1"
}
},
"playwright-core": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-0.11.1.tgz",
"integrity": "sha512-9xsSkXlglvHIAofyNInA1p3beOAOBMWHZgiuH99gX1R8VL6fTXgfWD7pIvt+rJhVMJWMDAyMXRo4TYtYtdspIg==",
"dev": true,
"requires": {
"debug": "^4.1.0",
"extract-zip": "^1.6.6",
"https-proxy-agent": "^3.0.0",
"jpeg-js": "^0.3.6",
"pngjs": "^3.4.0",
"progress": "^2.0.3",
"proxy-from-env": "^1.0.0",
"rimraf": "^3.0.2",
"uuid": "^3.4.0",
"ws": "^6.1.0"
},
"dependencies": {
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"dev": true,
"requires": {
"ms": "^2.1.1"
}
},
"https-proxy-agent": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz",
"integrity": "sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg==",
"dev": true,
"requires": {
"agent-base": "^4.3.0",
"debug": "^3.1.0"
},
"dependencies": {
"debug": {
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"dev": true,
"requires": {
"ms": "^2.1.1"
}
}
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
"dev": true
},
"rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
"dev": true,
"requires": {
"glob": "^7.1.3"
}
},
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
"dev": true
},
"ws": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz",
"integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==",
"dev": true,
"requires": {
"async-limiter": "~1.0.0"
}
}
}
},
"pn": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz",
"integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==",
"dev": true
},
"pngjs": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz",
"integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==",
"dev": true
},
"popper.js": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.15.0.tgz",
@ -10487,6 +10632,12 @@
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
"dev": true
},
"progress": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"dev": true
},
"promise-inflight": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
@ -10534,6 +10685,12 @@
"ipaddr.js": "1.9.0"
}
},
"proxy-from-env": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
"integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=",
"dev": true
},
"prr": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
@ -11221,6 +11378,12 @@
"psl": "^1.1.24",
"punycode": "^1.4.1"
}
},
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
"dev": true
}
}
},
@ -12111,6 +12274,14 @@
"requires": {
"faye-websocket": "^0.10.0",
"uuid": "^3.0.1"
},
"dependencies": {
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
"dev": true
}
}
},
"sockjs-client": {
@ -13194,6 +13365,14 @@
"https-proxy-agent": "^2.2.1",
"node-fetch": "^2.2.0",
"uuid": "^3.3.2"
},
"dependencies": {
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
"dev": true
}
}
},
"terser": {
@ -14066,9 +14245,9 @@
"dev": true
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
"version": "7.0.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.2.tgz",
"integrity": "sha512-vy9V/+pKG+5ZTYKf+VcphF5Oc6EFiu3W8Nv3P3zIh0EqVI80ZxOzuPfe9EHjkFNvf8+xuTHVeei4Drydlx4zjw==",
"dev": true
},
"v8-compile-cache": {
@ -14686,6 +14865,14 @@
"requires": {
"ansi-colors": "^3.0.0",
"uuid": "^3.3.2"
},
"dependencies": {
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
"dev": true
}
}
},
"webpack-sources": {
@ -14938,6 +15125,15 @@
"decamelize": "^1.2.0"
}
},
"yauzl": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
"integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=",
"dev": true,
"requires": {
"fd-slicer": "~1.0.1"
}
},
"yn": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",

14
web-console/package.json
View File

@ -35,12 +35,15 @@
"scripts": {
"compile": "./script/build",
"pretest": "./script/build",
"test": "npm run tslint && npm run sasslint && jest --silent 2>&1",
"test-ci": "npm run tslint && npm run sasslint && jest --coverage",
"jest": "jest --config jest.unit.config.js src",
"test-base": "npm run tslint && npm run sasslint && npm run jest",
"test": "npm run test-base -- --silent 2>&1",
"test-ci": "npm run test-base -- --coverage",
"test-e2e": "jest --config jest.e2e.config.js --detectOpenHandles e2e-tests",
"codecov": "codecov --disable=gcov -p ..",
"coverage": "jest --coverage",
"coverage": "jest --coverage src",
"update-snapshots": "jest -u",
"tslint": "./node_modules/.bin/tslint -c tslint.json --project tsconfig.json --formatters-dir ./node_modules/awesome-code-style/formatter 'src/**/*.ts?(x)'",
"tslint": "./node_modules/.bin/tslint -c tslint.json --project tsconfig.json --formatters-dir ./node_modules/awesome-code-style/formatter '{src,e2e-tests}/**/*.ts?(x)'",
"tslint-fix": "npm run tslint -- --fix",
"tslint-changed-only": "git diff --diff-filter=ACMR --cached --name-only | grep -E \\.tsx\\?$ | xargs ./node_modules/.bin/tslint -c tslint.json --project tsconfig.json --formatters-dir ./node_modules/awesome-code-style/formatter",
"tslint-fix-changed-only": "npm run tslint-changed-only -- --fix",
@ -107,6 +110,7 @@
"@types/react-router-dom": "^5.1.0",
"@types/react-splitter-layout": "^3.0.0",
"@types/react-table": "6.8.5",
"@types/uuid": "^7.0.2",
"autoprefixer": "^9.7.0",
"awesome-code-style": "^1.4.3",
"babel-loader": "^8.0.6",
@ -122,6 +126,7 @@
"license-checker": "^25.0.1",
"node-sass": "^4.13.1",
"node-sass-chokidar": "^1.4.0",
"playwright-chromium": "^0.11.1",
"postcss-cli": "^6.1.3",
"postcss-loader": "^3.0.0",
"postcss-preset-env": "^6.7.0",
@ -137,6 +142,7 @@
"tslint": "^5.20.0",
"tslint-loader": "^3.5.4",
"typescript": "^3.7.2",
"uuid": "^7.0.2",
"webpack": "^4.41.2",
"webpack-bundle-analyzer": "^3.6.0",
"webpack-cli": "^3.3.9",

139
web-console/script/druid Executable file
View File

@ -0,0 +1,139 @@
#!/bin/bash -x
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eu
set -o pipefail
# Private methods:
function _log() {
local msg="$1"
echo "[$(date)] -- ${msg}"
}
function _error() {
local msg="$1"
1>&2 _log "ERROR: $msg"
exit 1
}
function _get_code_root() {
git rev-parse --show-toplevel
}
function _get_maven_property_value() {
local property="$1";
(
cd "$(_get_code_root)" \
&& mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression="${property}" -q -DforceStdout \
| sed 's/%$//'
)
}
function _get_druid_version() {
_get_maven_property_value "project.version"
}
function _get_zookeeper_version() {
_get_maven_property_value "zookeeper.version"
}
function _download_zookeeper() {
local dest="$1"
local zk_version="$(_get_zookeeper_version)"
_log "Downloading zookeeper"
curl -s "https://archive.apache.org/dist/zookeeper/zookeeper-${zk_version}/zookeeper-${zk_version}.tar.gz" \
| tar xz \
&& rm -rf "$dest" \
&& mv "zookeeper-${zk_version}" "$dest" \
&& rm -f "zookeeper-${zk_version}"
}
function _build_distribution() {
_log "Building druid distribution"
(
cd "$(_get_code_root)" \
&& mvn -Pdist,skip-static-checks,skip-tests -Dmaven.javadoc.skip=true -q -T1C install \
&& cd distribution/target \
&& tar xzf "apache-druid-$(_get_druid_version)-bin.tar.gz"
)
}
function _wait_for_200_response() {
local url="$1"
local delay="$2"
local tries="$3"
local counter=0
_log "Waiting for 200 response from ${url}"
until $(curl --output /dev/null --silent --head --fail "${url}"); do
if [ ${counter} -eq ${tries} ];then
_error "Max of ${tries} tries exceeded"
fi
printf '.'
counter=$(($counter+1))
sleep "$delay"
done
_log "${url} is ready"
}
DRUID_PID_FILE=".druid.pid"
# Public methods:
function build() {
_log "Building druid"
_build_distribution \
&& _download_zookeeper "$(_get_code_root)/distribution/target/apache-druid-$(_get_druid_version)/zk"
_log "Done building druid"
}
function start() {
_log "Starting druid"
if [ -f "${DRUID_PID_FILE}" ]; then
_error "${DRUID_PID_FILE} exists with pid '$(<${DRUID_PID_FILE})'. Either shutdown druid or delete this file."
fi
"$(_get_code_root)/distribution/target/apache-druid-$(_get_druid_version)/bin/start-micro-quickstart" > /dev/null &
local pid="$!"
echo "$pid" > "$DRUID_PID_FILE"
_log "Druid started with pid ${pid}"
_wait_for_200_response "http://localhost:8888/unified-console.html" 3 20
}
function stop() {
_log "Stopping druid"
if [ -f "${DRUID_PID_FILE}" ]; then
local pid="$(<${DRUID_PID_FILE})"
kill "$pid"
rm "${DRUID_PID_FILE}"
_log "Stopped pid ${pid}"
else
_log "${DRUID_PID_FILE} file does not exist"
fi
}
$@

4
web-console/src/console-application.tsx
View File

@ -63,8 +63,8 @@ export class ConsoleApplication extends React.PureComponent<
timeout: 120000,
message: (
<>
It appears that the the service serving this console is not responding. The console will
not function at the moment
It appears that the service serving this console is not responding. The console will not
function at the moment.
</>
),
});

2
web-console/tsconfig.json
View File

@ -20,5 +20,5 @@
"jsx": "react",
"rootDirs": ["lib", "src"]
},
"include": ["src/**/*.ts", "src/**/*.tsx"]
"include": ["src/**/*.ts", "src/**/*.tsx", "e2e-tests/**/*.ts"]
}