From f83fab699ee3756d3a749961efa2dca41fad7841 Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Sat, 25 Jun 2022 13:43:59 -0700 Subject: [PATCH] Add IT-related changes pulled out of PR #12368 (#12673) This commit contains changes made to the existing ITs to support the new ITs. Changes: - Make the "custom node role" code usable by the new ITs. - Use flag `-DskipITs` to skips the integration tests but runs unit tests. - Use flag `-DskipUTs` skips unit tests but runs the "new" integration tests. - Expand the existing Druid profile, `-P skip-tests` to skip both ITs and UTs. --- integration-tests/README.md | 134 +++++++++++------- .../script/copy_resources_template.sh | 2 +- .../apache/druid/cli/CliCustomNodeRole.java | 3 +- .../testing/IntegrationTestingConfig.java | 13 ++ .../clients/SqlResourceTestClient.java | 1 - .../utils/AbstractTestQueryHelper.java | 13 +- .../utils/DruidClusterAdminClient.java | 2 +- .../org/testng/DruidTestRunnerFactory.java | 2 - ...eRoleWithOverrideCredentialsIndexTest.java | 2 +- pom.xml | 28 +++- 10 files changed, 137 insertions(+), 63 deletions(-) diff --git a/integration-tests/README.md b/integration-tests/README.md index e80151d92f6..59af6f37636 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -17,26 +17,26 @@ ~ under the License. --> -Integration Testing +Integration Testing =================== To run integration tests, you have to specify the druid cluster the -tests should use. +tests should use. Druid comes with the mvn profile integration-tests for setting up druid running in docker containers, and using that cluster to run the integration tests. To use a druid cluster that is already running, use the -mvn profile int-tests-config-file, which uses a configuration file +mvn profile int-tests-config-file, which uses a configuration file describing the cluster. -Integration Testing Using Docker +Integration Testing Using Docker ------------------- -Before starting, if you don't already have docker on your machine, install it as described on -[Docker installation instructions](https://docs.docker.com/install/). Ensure that you -have at least 4GiB of memory allocated to the docker engine. (You can verify it +Before starting, if you don't already have docker on your machine, install it as described on +[Docker installation instructions](https://docs.docker.com/install/). Ensure that you +have at least 4GiB of memory allocated to the docker engine. (You can verify it under Preferences > Resources > Advanced.) Also set the `DOCKER_IP` @@ -52,49 +52,86 @@ Optionally, you can also set `APACHE_ARCHIVE_MIRROR_HOST` to override `https://a export APACHE_ARCHIVE_MIRROR_HOST=https://example.com/remote-generic-repo ``` -## Running tests againt auto brought up Docker containers +## Running tests against auto brought up Docker containers -> NOTE: This section describes how to start integration tests against docker containers which will be brought up automatically by following commands. -If you want to buid docker images and run tests separately, see the next section. +This section describes how to start integration tests against Docker containers which will be brought up automatically by following commands. +If you want to build Docker images and run tests separately, see the next section. + +To run all tests from a test group using Docker and Maven run the following command: -To run all tests from a test group using docker and mvn run the following command: -(list of test groups can be found at `integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java`) ```bash mvn verify -P integration-tests -Dgroups= ``` -To run only a single test using mvn run the following command: +The list of test groups can be found at +`integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java`. + +### Run a single test + +To run only a single test using Maven: + ```bash mvn verify -P integration-tests -Dgroups= -Dit.test= ``` -The test group should always be set, as certain test setup and cleanup tasks are based on the test group. You can find -the test group for a given test as an annotation in the respective test class. -Add `-rf :druid-integration-tests` when running integration tests for the second time or later without changing +Parameters: + +* Test Group: Required, as certain test tasks for setup and cleanup are based on the test group. You can find +the test group for a given test as an annotation in the respective test class. A list of test groups can be found at +`integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java`. The annotation uses a string +constant defined in `TestNGGroup.java`, be sure to use the constant value, not name. For example, +if your test has the the annotation: `@Test(groups = TestNGGroup.BATCH_INDEX)` then use the argument +`-Dgroups=batch-index`. + +* Test Name: Use the fully-qualified class name. For example, `org.apache.druid.tests.BATCH_INDEX`. + +* Add `-pl :druid-integration-tests` when running integration tests for the second time or later without changing the code of core modules in between to skip up-to-date checks for the whole module dependency tree. -Integration tests can also be run with either Java 8 or Java 11 by adding `-Djvm.runtime=#` to mvn command, where `#` +* Integration tests can also be run with either Java 8 or Java 11 by adding `-Djvm.runtime=#` to the `mvn` command, where `#` can either be 8 or 11. -Druid's configuration (using Docker) can be overrided by providing `-Doverride.config.path=`. +* Druid's configuration (using Docker) can be overridden by providing `-Doverride.config.path=`. The file must contain one property per line, the key must start with `druid_` and the format should be snake case. -Note that when bringing up docker containers through mvn and -Doverride.config.path is provided, additional -Druid routers for security group integration test (permissive tls, no client auth tls, custom check tls) will not be started. +Note that when bringing up Docker containers through Maven and `-Doverride.config.path` is provided, additional +Druid routers for security group integration test (permissive tls, no client auth tls, custom check tls) will not be started. + +### Debugging test runs + +The integration test process is fragile and can fail for many reasons when run on your machine. +Here are some suggestions. + +#### Workround for failed builds + +Sometimes the command above may fail for reasons unrelated to the changes you wish to test. +In such cases, a workaround is to build the code first, then use the next section to run +individual tests. To build: + +```bash +mvn clean package -P integration-tests -Pskip-static-checks -Pskip-tests -Dmaven.javadoc.skip=true -T1.0C -nsu +``` + +#### Keep the local Maven cache fresh + +As you work with issues, you may be tempted to reuse already-built jars. That only works for about 24 hours, +after which Maven will helpfully start downloading snapshot jars from an upstream repository. +This is, unfortunately, a feature of the build scripts. The `-nsu` option above tries to force +Maven to only look locally for snapshot jars. ## Running tests against mannually brought up Docker containers 1. Build docker images. - + From root module run maven command, run the following command: ```bash mvn clean install -pl integration-tests -P integration-tests -Ddocker.run.skip=true -Dmaven.test.skip=true -Ddocker.build.hadoop=true - ``` - + ``` + > **NOTE**: `-Ddocker.build.hadoop=true` is optional if you don't run tests against Hadoop. 2. Choose a docker-compose file to start containers. - There are a few different Docker compose yamls located in "docker" folder that could be used to start containers for different tests. + There are a few different Docker compose yamls located in "docker" folder that could be used to start containers for different tests. - To start basic Druid cluster (skip this if running Druid cluster with override configs): ```bash @@ -105,19 +142,19 @@ Druid routers for security group integration test (permissive tls, no client aut ```bash OVERRIDE_ENV= docker-compose -f docker-compose.yml up ``` - + - To start tests against Hadoop ```bash docker-compose -f docker-compose.druid-hadoop.yml up ``` - + - To start tests againt security group ```bash docker-compose -f docker-compose.yml -f docker-compose.security.yml up ``` 3. Run tests. - + Execute the following command from root module, where `` is the class name of a test, such as ITIndexerTest. ```bash mvn verify -P integration-tests -pl integration-tests -Ddocker.build.skip=true -Ddocker.run.skip=true -Dit.test= @@ -178,14 +215,13 @@ The values shown above are for the default docker compose cluster. For other clu ``` - docker-compose.druid-hadoop.yml - + For starting Apache Hadoop 2.8.5 cluster with the same setup as the Druid tutorial. ```bash docker-compose -f docker-compose.druid-hadoop.yml up ``` - ## Tips & tricks for debugging and developing integration tests ### Useful mvn command flags @@ -262,7 +298,7 @@ Make sure that you have at least 6GiB of memory available before you run the tes To run tests on any druid cluster that is already running, create a configuration file: - { + { "broker_host": "", "broker_port": "", "router_host": "", @@ -282,7 +318,7 @@ Set the environment variable `CONFIG_FILE` to the name of the configuration file export CONFIG_FILE= ``` -To run all tests from a test group using mvn run the following command: +To run all tests from a test group using mvn run the following command: (list of test groups can be found at integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java) ```bash mvn verify -P int-tests-config-file -Dgroups= @@ -297,10 +333,10 @@ Running a Test That Uses Cloud ------------------- The integration test that indexes from Cloud or uses Cloud as deep storage is not run as part of the integration test run discussed above. Running these tests requires the user to provide -their own Cloud. +their own Cloud. Currently, the integration test supports Amazon Kinesis, Google Cloud Storage, Amazon S3, and Microsoft Azure. -These can be run by providing "kinesis-index", "kinesis-data-format", "gcs-deep-storage", "s3-deep-storage", or "azure-deep-storage" +These can be run by providing "kinesis-index", "kinesis-data-format", "gcs-deep-storage", "s3-deep-storage", or "azure-deep-storage" to -Dgroups for Amazon Kinesis, Google Cloud Storage, Amazon S3, and Microsoft Azure respectively. Note that only one group should be run per mvn command. @@ -309,13 +345,13 @@ For all the Cloud Integration tests, the following will also need to be provided integration-tests/docker/environment-configs/override-examples/ directory for env vars to provide for each Cloud. For Amazon Kinesis, the following will also need to be provided: -1) Provide -Ddruid.test.config.streamEndpoint= with the endpoint of your stream set. +1) Provide -Ddruid.test.config.streamEndpoint= with the endpoint of your stream set. For example, kinesis.us-east-1.amazonaws.com For Google Cloud Storage, Amazon S3, and Microsoft Azure, the following will also need to be provided: -1) Set the bucket and path for your test data. This can be done by setting -Ddruid.test.config.cloudBucket and +1) Set the bucket and path for your test data. This can be done by setting -Ddruid.test.config.cloudBucket and -Ddruid.test.config.cloudPath in the mvn command or setting "cloud_bucket" and "cloud_path" in the config file. -2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and wikipedia_index_data3.json +2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and wikipedia_index_data3.json located in integration-tests/src/test/resources/data/batch_index/json to your Cloud storage at the location set in step 1. For Google Cloud Storage, in addition to the above, you will also have to: @@ -326,21 +362,21 @@ For example, to run integration test for Google Cloud Storage: mvn verify -P integration-tests -Dgroups=gcs-deep-storage -Doverride.config.path= -Dresource.file.dir.path= -Ddruid.test.config.cloudBucket=test-bucket -Ddruid.test.config.cloudPath=test-data-folder/ ``` - + Running a Test That Uses Hadoop ------------------- The integration test that indexes from hadoop is not run as part of the integration test run discussed above. This is because druid test clusters might not, in general, have access to hadoop. -This also applies to integration test that uses Hadoop HDFS as an inputSource or as a deep storage. +This also applies to integration test that uses Hadoop HDFS as an inputSource or as a deep storage. To run integration test that uses Hadoop, you will have to run a Hadoop cluster. This can be done in two ways: 1) Run Druid Docker test clusters with Hadoop container by passing -Dstart.hadoop.docker=true to the mvn command. If you have not already built the hadoop image, you will also need to add -Ddocker.build.hadoop=true to the mvn command. 2) Run your own Druid + Hadoop cluster and specified Hadoop configs in the configuration file (CONFIG_FILE). -Currently, hdfs-deep-storage and other -deep-storage integration test groups can only be run with +Currently, hdfs-deep-storage and other -deep-storage integration test groups can only be run with Druid Docker test clusters by passing -Dstart.hadoop.docker=true to start Hadoop container. -You will also have to provide -Doverride.config.path= with your Druid's Hadoop configs set. +You will also have to provide -Doverride.config.path= with your Druid's Hadoop configs set. See integration-tests/docker/environment-configs/override-examples/hdfs directory for example. Note that if the integration test you are running also uses other cloud extension (S3, Azure, GCS), additional credentials/configs may need to be set in the same file as your Druid's Hadoop configs set. @@ -383,9 +419,9 @@ do the following instead of running the tests using mvn: On a machine that can do mvn builds: ```bash -cd druid +cd druid mvn clean package -cd integration_tests +cd integration_tests mvn dependency:copy-dependencies package ``` @@ -450,23 +486,23 @@ Refer ITIndexerTest as an example on how to use dependency Injection ### Running test methods in parallel -By default, test methods in a test class will be run in sequential order one at a time. Test methods for a given test +By default, test methods in a test class will be run in sequential order one at a time. Test methods for a given test class can be set to run in parallel (multiple test methods of each class running at the same time) by excluding -the given class/package from the "AllSerializedTests" test tag section and including it in the "AllParallelizedTests" +the given class/package from the "AllSerializedTests" test tag section and including it in the "AllParallelizedTests" test tag section in integration-tests/src/test/resources/testng.xml. TestNG uses two parameters, i.e., `thread-count` and `data-provider-thread-count`, for parallel test execution, which are both set to 2 for Druid integration tests. For test using parallel execution with data provider, you will also need to set `@DataProvider(parallel = true)` on your data provider method in your test class. Note that for test using parallel execution with data provider, the test -class does not need to be in the "AllParallelizedTests" test tag section and if it is in the "AllParallelizedTests" +class does not need to be in the "AllParallelizedTests" test tag section and if it is in the "AllParallelizedTests" test tag section it will actually be run with `thread-count` times `data-provider-thread-count` threads. You may want to modify those values for faster execution. See https://testng.org/doc/documentation-main.html#parallel-running and https://testng.org/doc/documentation-main.html#parameters-dataproviders for details. Please be mindful when adding tests to the "AllParallelizedTests" test tag that the tests can run in parallel with other tests from the same class at the same time. i.e. test does not modify/restart/stop the druid cluster or other dependency containers, -test does not use excessive memory starving other concurent task, test does not modify and/or use other task, -supervisor, datasource it did not create. +test does not use excessive memory starving other concurent task, test does not modify and/or use other task, +supervisor, datasource it did not create. ### Limitation of Druid cluster in Travis environment @@ -474,6 +510,6 @@ By default, integration tests are run in Travis environment on commits made to o required to pass for a PR to be elligible to be merged. Here are known issues and limitations to the Druid docker cluster running in Travis machine that may cause the tests to fail: - Number of concurrent running tasks. Although the default Druid cluster config sets the maximum number of tasks (druid.worker.capacity) to 10, -the actual maximum can be lowered depending on the type of the tasks. For example, running 2 range partitioning compaction tasks with 2 subtasks each -(for a total of 6 tasks) concurrently can cause the cluster to intermittently fail. This can cause the Travis job to become stuck until it timeouts (50 minutes) +the actual maximum can be lowered depending on the type of the tasks. For example, running 2 range partitioning compaction tasks with 2 subtasks each +(for a total of 6 tasks) concurrently can cause the cluster to intermittently fail. This can cause the Travis job to become stuck until it timeouts (50 minutes) and/or terminates after 10 mins of not receiving new output. diff --git a/integration-tests/script/copy_resources_template.sh b/integration-tests/script/copy_resources_template.sh index 7dab593c752..ef35da2ad9f 100755 --- a/integration-tests/script/copy_resources_template.sh +++ b/integration-tests/script/copy_resources_template.sh @@ -30,7 +30,7 @@ cp -R docker $SHARED_DIR/docker pushd ../ rm -rf distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin -mvn -DskipTests -T1C -Danimal.sniffer.skip=true -Dcheckstyle.skip=true -Ddruid.console.skip=true -Denforcer.skip=true -Dforbiddenapis.skip=true -Dmaven.javadoc.skip=true -Dpmd.skip=true -Dspotbugs.skip=true install -Pintegration-test +mvn -P skip-static-checks,skip-tests -T1C -Danimal.sniffer.skip=true -Dcheckstyle.skip=true -Ddruid.console.skip=true -Denforcer.skip=true -Dforbiddenapis.skip=true -Dmaven.javadoc.skip=true -Dpmd.skip=true -Dspotbugs.skip=true install -Pintegration-test mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib $SHARED_DIR/docker/lib mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions $SHARED_DIR/docker/extensions popd diff --git a/integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java b/integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java index be8a5be5094..5542bbea95e 100644 --- a/integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java +++ b/integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java @@ -66,6 +66,7 @@ public class CliCustomNodeRole extends ServerRunnable public static final String SERVICE_NAME = "custom-node-role"; public static final int PORT = 9301; public static final int TLS_PORT = 9501; + public static final NodeRole NODE_ROLE = new NodeRole(CliCustomNodeRole.SERVICE_NAME); public CliCustomNodeRole() { @@ -75,7 +76,7 @@ public class CliCustomNodeRole extends ServerRunnable @Override protected Set getNodeRoles(Properties properties) { - return ImmutableSet.of(new NodeRole(CliCustomNodeRole.SERVICE_NAME)); + return ImmutableSet.of(NODE_ROLE); } @Override diff --git a/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfig.java b/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfig.java index 25d958018ec..e52df0bb2bd 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfig.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfig.java @@ -23,6 +23,19 @@ import javax.annotation.Nullable; import java.util.Map; /** + * Configuration for tests. Opinionated about the shape of the cluster: + * there is one or two Coordinators or Overlords, zero or one of + * everything else. + *

+ * To work in Docker (and K8s) there are two methods per host: + * {@code getHost()} which returns the host as seen from + * the test machine (meaning the proxy host), and + * {@code getInternalHost()} which returns the name of + * the host as seen by itself and other services: the host published + * in ZK, which is the host known to the Docker/K8s overlay network. + *

+ * The {@code getUrl()} methods return URLs relative to + * the test, using the proxy host for Docker and K8s. */ public interface IntegrationTestingConfig { diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/SqlResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/SqlResourceTestClient.java index d41c9c666a4..8f126be2136 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/clients/SqlResourceTestClient.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/SqlResourceTestClient.java @@ -30,7 +30,6 @@ import javax.ws.rs.core.MediaType; public class SqlResourceTestClient extends AbstractQueryResourceTestClient { - @Inject SqlResourceTestClient( ObjectMapper jsonMapper, diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/AbstractTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/AbstractTestQueryHelper.java index fce45a8cb7c..8c7bcb0208e 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/utils/AbstractTestQueryHelper.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/AbstractTestQueryHelper.java @@ -41,7 +41,6 @@ import java.util.function.Function; public abstract class AbstractTestQueryHelper { - public static final Logger LOG = new Logger(TestQueryHelper.class); protected final AbstractQueryResourceTestClient queryClient; @@ -54,7 +53,7 @@ public abstract class AbstractTestQueryHelper queryClient, IntegrationTestingConfig config ) { @@ -70,7 +69,7 @@ public abstract class AbstractTestQueryHelper queryClient, String broker, String brokerTLS, String router, @@ -103,9 +102,13 @@ public abstract class AbstractTestQueryHelper 3 + + + false + false @@ -1169,6 +1176,7 @@ org/apache/druid/cli/Cli* + org/apache/druid/cli/GuiceRunnable.class org/apache/druid/cli/*JettyServerInitializer* org/apache/druid/server/initialization/*Module* org/apache/druid/server/initialization/jetty/*Module* @@ -1183,6 +1191,13 @@ org/apache/druid/benchmark/**/* org/apache/druid/**/*Benchmark* org/apache/druid/testing/**/* + + org/apache/druid/server/coordination/ServerManagerForQueryErrorTest.class + org/apache/druid/guice/SleepModule.class + org/apache/druid/guice/CustomNodeRoleClientModule.class + org/apache/druid/cli/CustomNodeRoleCommandCreator.class + org/apache/druid/cli/QueryRetryTestCommandCreator.class org/apache/druid/query/TruncatedResponseContextException.class @@ -1513,6 +1528,9 @@ -Ddruid.indexing.doubleStorage=double + + ${skipUTs} false true @@ -1829,7 +1847,7 @@ docker/service-supervisords/*.conf target/** licenses/** - **/test/resources/** + **/test/resources/** **/derby.log **/jvm.config **/*.avsc @@ -1837,6 +1855,8 @@ **/*.json **/*.parq **/*.parquet + **/*.pmd + **/*.pmdruleset.xml **/docker/schema-registry/* LICENSE LICENSE.BINARY @@ -1911,8 +1931,12 @@ skip-tests + + false + - true + true + true true