From f69e0591b06ae095a9271f3fda24792436fb66f3 Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Thu, 6 Nov 2014 23:53:24 +0530 Subject: [PATCH] working integration tests --- integration-tests/README.md | 90 + integration-tests/docker/Dockerfile | 71 + integration-tests/docker/broker.conf | 29 + integration-tests/docker/coordinator.conf | 23 + integration-tests/docker/historical.conf | 27 + .../docker/metadata-storage.conf | 6 + integration-tests/docker/middlemanager.conf | 29 + integration-tests/docker/overlord.conf | 25 + integration-tests/docker/router.conf | 20 + integration-tests/docker/sample-data.sql | 5 + integration-tests/docker/supervisord.conf | 6 + integration-tests/docker/zookeeper.conf | 5 + integration-tests/pom.xml | 151 ++ integration-tests/run_cluster.sh | 49 + .../druid/testing/DockerConfigProvider.java | 67 + .../testing/IntegrationTestingConfig.java | 33 + .../IntegrationTestingConfigProvider.java | 32 + .../CoordinatorResourceTestClient.java | 146 ++ .../EventReceiverFirehoseTestClient.java | 135 + .../clients/OverlordResourceTestClient.java | 213 ++ .../clients/QueryResourceTestClient.java | 98 + .../testing/clients/TaskResponseObject.java | 68 + .../druid/testing/guice/DruidTestModule.java | 54 + .../testing/guice/DruidTestModuleFactory.java | 63 + .../utils/FromFileTestQueryHelper.java | 79 + .../testing/utils/QueryResultVerifier.java | 49 + .../druid/testing/utils/QueryWithResults.java | 64 + .../io/druid/testing/utils/RetryUtil.java | 76 + .../testing/utils/ServerDiscoveryUtil.java | 64 + .../org/testng/DruidTestRunnerFactory.java | 150 ++ .../src/main/java/org/testng/TestNG.java | 2252 +++++++++++++++++ .../java/org/testng/remote/RemoteTestNG.java | 336 +++ .../tests/indexer/AbstractIndexerTest.java | 85 + .../io/druid/tests/indexer/ITIndexerTest.java | 81 + .../indexer/ITRealtimeIndexTaskTest.java | 142 ++ .../druid/tests/indexer/ITUnionQueryTest.java | 172 ++ .../druid/tests/query/ITTwitterQueryTest.java | 65 + .../tests/query/ITWikipediaQueryTest.java | 65 + .../src/test/resources/indexer/select.query | 19 + .../resources/indexer/select_reindex.query | 19 + .../test/resources/indexer/union_queries.json | 564 +++++ .../resources/indexer/union_select_query.json | 121 + .../indexer/wikipedia_index_data.json | 5 + .../indexer/wikipedia_index_queries.json | 16 + .../indexer/wikipedia_index_task.json | 59 + .../wikipedia_realtime_index_task.json | 71 + .../indexer/wikipedia_reindex_task.json | 62 + .../queries/twitterstream_queries.json | 780 ++++++ .../queries/wikipedia_editstream_queries.json | 1063 ++++++++ .../src/test/resources/testng.xml | 10 + integration-tests/stop_cluster.sh | 5 + pom.xml | 6 + 52 files changed, 7925 insertions(+) create mode 100644 integration-tests/README.md create mode 100644 integration-tests/docker/Dockerfile create mode 100644 integration-tests/docker/broker.conf create mode 100644 integration-tests/docker/coordinator.conf create mode 100644 integration-tests/docker/historical.conf create mode 100644 integration-tests/docker/metadata-storage.conf create mode 100644 integration-tests/docker/middlemanager.conf create mode 100644 integration-tests/docker/overlord.conf create mode 100644 integration-tests/docker/router.conf create mode 100644 integration-tests/docker/sample-data.sql create mode 100644 integration-tests/docker/supervisord.conf create mode 100644 integration-tests/docker/zookeeper.conf create mode 100644 integration-tests/pom.xml create mode 100755 integration-tests/run_cluster.sh create mode 100644 integration-tests/src/main/java/io/druid/testing/DockerConfigProvider.java create mode 100644 integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfig.java create mode 100644 integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfigProvider.java create mode 100644 integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java create mode 100644 integration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java create mode 100644 integration-tests/src/main/java/io/druid/testing/clients/OverlordResourceTestClient.java create mode 100644 integration-tests/src/main/java/io/druid/testing/clients/QueryResourceTestClient.java create mode 100644 integration-tests/src/main/java/io/druid/testing/clients/TaskResponseObject.java create mode 100644 integration-tests/src/main/java/io/druid/testing/guice/DruidTestModule.java create mode 100644 integration-tests/src/main/java/io/druid/testing/guice/DruidTestModuleFactory.java create mode 100644 integration-tests/src/main/java/io/druid/testing/utils/FromFileTestQueryHelper.java create mode 100644 integration-tests/src/main/java/io/druid/testing/utils/QueryResultVerifier.java create mode 100644 integration-tests/src/main/java/io/druid/testing/utils/QueryWithResults.java create mode 100644 integration-tests/src/main/java/io/druid/testing/utils/RetryUtil.java create mode 100644 integration-tests/src/main/java/io/druid/testing/utils/ServerDiscoveryUtil.java create mode 100644 integration-tests/src/main/java/org/testng/DruidTestRunnerFactory.java create mode 100644 integration-tests/src/main/java/org/testng/TestNG.java create mode 100644 integration-tests/src/main/java/org/testng/remote/RemoteTestNG.java create mode 100644 integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java create mode 100644 integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java create mode 100644 integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java create mode 100644 integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java create mode 100644 integration-tests/src/test/java/io/druid/tests/query/ITTwitterQueryTest.java create mode 100644 integration-tests/src/test/java/io/druid/tests/query/ITWikipediaQueryTest.java create mode 100644 integration-tests/src/test/resources/indexer/select.query create mode 100644 integration-tests/src/test/resources/indexer/select_reindex.query create mode 100644 integration-tests/src/test/resources/indexer/union_queries.json create mode 100644 integration-tests/src/test/resources/indexer/union_select_query.json create mode 100644 integration-tests/src/test/resources/indexer/wikipedia_index_data.json create mode 100644 integration-tests/src/test/resources/indexer/wikipedia_index_queries.json create mode 100644 integration-tests/src/test/resources/indexer/wikipedia_index_task.json create mode 100644 integration-tests/src/test/resources/indexer/wikipedia_realtime_index_task.json create mode 100644 integration-tests/src/test/resources/indexer/wikipedia_reindex_task.json create mode 100644 integration-tests/src/test/resources/queries/twitterstream_queries.json create mode 100644 integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json create mode 100644 integration-tests/src/test/resources/testng.xml create mode 100755 integration-tests/stop_cluster.sh diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 00000000000..abeae63a0aa --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,90 @@ +Integration Testing +========================= + +## Installing Docker and Running + +Please refer to instructions at [https://github.com/druid-io/docker-druid/blob/master/docker-install.md](https://github.com/druid-io/docker-druid/blob/master/docker-install.md) + +Instead of running +``` +boot2docker init +``` + +run instead +``` +boot2docker init -m 6000 +``` + +Make sure that you have at least 6GB of memory available before you run the tests. + +Set the docker ip via: +``` +export DOCKER_IP=$(boot2docker ip 2>/dev/null) +``` + +Verify that docker is running by issuing the following command: + +``` +docker info +``` + +Running Integration tests +========================= + +## Running tests using mvn + +To run all the tests using mvn run the following command - +''''' + mvn verify -P integration-tests +''''' + +To run only a single test using mvn run following command - +''''' + mvn verify -P integration-tests -Dit.test= +''''' + + +Writing a New Test +=============== + +## What should we cover in integration tests + +For every end-user functionality provided by druid we should have an integration-test verifying the correctness. + +## Rules to be followed while writing a new integration test + +### Every Integration Test must follow these rules + +1) Name of the test must start with a prefix "IT" +2) A test should be independent of other tests +3) Tests are to be written in TestNG style ([http://testng.org/doc/documentation-main.html#methods](http://testng.org/doc/documentation-main.html#methods)) +4) If a test loads some data it is the responsibility of the test to clean up the data from the cluster + +### How to use Guice Dependency Injection in a test + +A test can access different helper and utility classes provided by test-framework in order to access Coordinator,Broker etc.. +To mark a test be able to use Guice Dependency Injection - +Annotate the test class with the below annotation + + ''''''' + @Guice(moduleFactory = DruidTestModuleFactory.class) + ''''''' +This will tell the test framework that the test class needs to be constructed using guice. + +### Helper Classes provided + +1) IntegrationTestingConfig - configuration of the test +2) CoordinatorResourceTestClient - httpclient for coordinator endpoints +3) OverlordResourceTestClient - httpclient for indexer endpoints +4) QueryResourceTestClient - httpclient for broker endpoints + +### Static Utility classes + +1) RetryUtil - provides methods to retry an operation until it succeeds for configurable no. of times +2) FromFileTestQueryHelper - reads queries with expected results from file and executes them and verifies the results using ResultVerifier + +Refer ITIndexerTest as an example on how to use dependency Injection + +TODOS +======================= +1) Remove the patch for TestNG after resolution of Surefire-622 diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile new file mode 100644 index 00000000000..8f03b95cd0d --- /dev/null +++ b/integration-tests/docker/Dockerfile @@ -0,0 +1,71 @@ +FROM ubuntu:14.04 + +# Add Java 7 repository +RUN apt-get update +RUN apt-get install -y software-properties-common +RUN apt-add-repository -y ppa:webupd8team/java +RUN apt-get update + +# Oracle Java 7 +RUN echo oracle-java-7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections +RUN apt-get install -y oracle-java7-installer +RUN apt-get install -y oracle-java7-set-default + +# MySQL (Metadata store) +RUN apt-get install -y mysql-server + +# Supervisor +RUN apt-get install -y supervisor + +# Maven +RUN wget -q -O - http://www.us.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz | tar -xzf - -C /usr/local +RUN ln -s /usr/local/apache-maven-3.2.1 /usr/local/apache-maven +RUN ln -s /usr/local/apache-maven/bin/mvn /usr/local/bin/mvn + +# Zookeeper +RUN wget -q -O - http://www.us.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz | tar -xzf - -C /usr/local +RUN cp /usr/local/zookeeper-3.4.6/conf/zoo_sample.cfg /usr/local/zookeeper-3.4.6/conf/zoo.cfg +RUN ln -s /usr/local/zookeeper-3.4.6 /usr/local/zookeeper + +# git +RUN apt-get install -y git + +# Druid system user +RUN adduser --system --group --no-create-home druid +RUN mkdir -p /var/lib/druid +RUN chown druid:druid /var/lib/druid + +# Add druid jars +ADD lib/* /usr/local/druid/lib/ + +WORKDIR / + +# Setup metadata store +RUN /etc/init.d/mysql start && echo "GRANT ALL ON druid.* TO 'druid'@'%' IDENTIFIED BY 'diurd'; CREATE database druid;" | mysql -u root && /etc/init.d/mysql stop + +# Add sample data +RUN /etc/init.d/mysql start && java -Ddruid.metadata.storage.type=mysql -cp "/usr/local/druid/lib/*" io.druid.cli.Main tools metadata-init --connectURI="jdbc:mysql://localhost:3306/druid" --user=druid --password=diurd && /etc/init.d/mysql stop +ADD sample-data.sql sample-data.sql +RUN /etc/init.d/mysql start && cat sample-data.sql | mysql -u root druid && /etc/init.d/mysql stop + +# Setup supervisord +ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +# Clean up +RUN apt-get clean && rm -rf /tmp/* /var/tmp/* + +# Expose ports: +# - 8081: HTTP (coordinator) +# - 8082: HTTP (broker) +# - 8083: HTTP (historical) +# - 3306: MySQL +# - 2181 2888 3888: ZooKeeper +# - 8100 8101 8102 8103 8104 : peon ports +EXPOSE 8081 +EXPOSE 8082 +EXPOSE 8083 +EXPOSE 3306 +EXPOSE 2181 2888 3888 +EXPOSE 8100 8101 8102 8103 8104 +WORKDIR /var/lib/druid +ENTRYPOINT export HOST_IP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/integration-tests/docker/broker.conf b/integration-tests/docker/broker.conf new file mode 100644 index 00000000000..63b6af12e3c --- /dev/null +++ b/integration-tests/docker/broker.conf @@ -0,0 +1,29 @@ +[program:druid-broker] +command=java + -server + -Xmx1g + -Xms1g + -XX:NewSize=500m + -XX:MaxNewSize=500m + -XX:+UseConcMarkSweepGC + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOST_IP)s + -Ddruid.zk.service.host=druid-zookeeper + -Ddruid.processing.buffer.sizeBytes=75000000 + -Ddruid.server.http.numThreads=100 + -Ddruid.processing.numThreads=1 + -Ddruid.broker.http.numConnections=30 + -Ddruid.broker.http.readTimeout=PT5M + -Ddruid.broker.cache.useCache=true + -Ddruid.broker.cache.populateCache=true + -Ddruid.cache.type=local + -Ddruid.cache.sizeInBytes=40000000 + -cp /usr/local/druid/lib/* + io.druid.cli.Main server broker +redirect_stderr=true +autorestart=false +priority=100 +stdout_logfile=/shared/logs/broker.log diff --git a/integration-tests/docker/coordinator.conf b/integration-tests/docker/coordinator.conf new file mode 100644 index 00000000000..60cf6b97362 --- /dev/null +++ b/integration-tests/docker/coordinator.conf @@ -0,0 +1,23 @@ +[program:druid-coordinator] +command=java + -server + -Xmx128m + -Xms128m + -XX:+UseConcMarkSweepGC + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOST_IP)s + -Ddruid.metadata.storage.type=mysql + -Ddruid.metadata.storage.connector.connectURI=jdbc:mysql://druid-metadata-storage/druid + -Ddruid.metadata.storage.connector.user=druid + -Ddruid.metadata.storage.connector.password=diurd + -Ddruid.zk.service.host=druid-zookeeper + -Ddruid.coordinator.startDelay=PT5S + -cp /usr/local/druid/lib/* + io.druid.cli.Main server coordinator +redirect_stderr=true +priority=100 +autorestart=false +stdout_logfile=/shared/logs/coordinator.log diff --git a/integration-tests/docker/historical.conf b/integration-tests/docker/historical.conf new file mode 100644 index 00000000000..ab02bbc664a --- /dev/null +++ b/integration-tests/docker/historical.conf @@ -0,0 +1,27 @@ +[program:druid-historical] +command=java + -server + -Xmx1500m + -Xms1500m + -XX:NewSize=750m + -XX:MaxNewSize=750m + -XX:+UseConcMarkSweepGC + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOST_IP)s + -Ddruid.zk.service.host=druid-zookeeper + -Ddruid.s3.accessKey=AKIAIMKECRUYKDQGR6YQ + -Ddruid.s3.secretKey=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 + -Ddruid.processing.buffer.sizeBytes=75000000 + -Ddruid.processing.numThreads=3 + -Ddruid.server.http.numThreads=100 + -Ddruid.segmentCache.locations="[{\"path\":\"/shared/druid/indexCache\",\"maxSize\":5000000000}]" + -Ddruid.server.maxSize=5000000000 + -cp /usr/local/druid/lib/* + io.druid.cli.Main server historical +redirect_stderr=true +priority=100 +autorestart=false +stdout_logfile=/shared/logs/historical.log diff --git a/integration-tests/docker/metadata-storage.conf b/integration-tests/docker/metadata-storage.conf new file mode 100644 index 00000000000..eb60e214665 --- /dev/null +++ b/integration-tests/docker/metadata-storage.conf @@ -0,0 +1,6 @@ +[program:mysql] +command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe + --bind-address=0.0.0.0 +user=mysql +priority=0 +stdout_logfile=/shared/logs/mysql.log diff --git a/integration-tests/docker/middlemanager.conf b/integration-tests/docker/middlemanager.conf new file mode 100644 index 00000000000..cf1436182e7 --- /dev/null +++ b/integration-tests/docker/middlemanager.conf @@ -0,0 +1,29 @@ +[program:druid-middlemanager] +command=java + -server + -Xmx64m + -Xms64m + -XX:+UseConcMarkSweepGC + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOST_IP)s + -Ddruid.zk.service.host=druid-zookeeper + -Ddruid.indexer.logs.directory=/shared/tasklogs + -Ddruid.storage.storageDirectory=/shared/storage + -Ddruid.indexer.runner.javaOpts=-server -Xmx256m -Xms256m -XX:NewSize=128m -XX:MaxNewSize=128m -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps + -Ddruid.indexer.fork.property.druid.processing.buffer.sizeBytes=75000000 + -Ddruid.indexer.fork.property.druid.processing.numThreads=1 + -Ddruid.indexer.fork.server.http.numThreads=100 + -Ddruid.s3.accessKey=AKIAIMKECRUYKDQGR6YQ + -Ddruid.s3.secretKey=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 + -Ddruid.worker.ip=%(ENV_HOST_IP)s + -Ddruid.selectors.indexing.serviceName=druid:overlord + -Ddruid.indexer.task.chathandler.type=announce + -cp /usr/local/druid/lib/* + io.druid.cli.Main server middleManager +redirect_stderr=true +priority=100 +autorestart=false +stdout_logfile=/shared/logs/middlemanager.log diff --git a/integration-tests/docker/overlord.conf b/integration-tests/docker/overlord.conf new file mode 100644 index 00000000000..a0d436c5a02 --- /dev/null +++ b/integration-tests/docker/overlord.conf @@ -0,0 +1,25 @@ +[program:druid-overlord] +command=java + -server + -Xmx128m + -Xms128m + -XX:+UseConcMarkSweepGC + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOST_IP)s + -Ddruid.metadata.storage.type=mysql + -Ddruid.metadata.storage.connector.connectURI=jdbc:mysql://druid-metadata-storage/druid + -Ddruid.metadata.storage.connector.user=druid + -Ddruid.metadata.storage.connector.password=diurd + -Ddruid.zk.service.host=druid-zookeeper + -Ddruid.indexer.storage.type=metadata + -Ddruid.indexer.logs.directory=/shared/tasklogs + -Ddruid.indexer.runner.type=remote + -cp /usr/local/druid/lib/* + io.druid.cli.Main server overlord +redirect_stderr=true +priority=100 +autorestart=false +stdout_logfile=/shared/logs/overlord.log diff --git a/integration-tests/docker/router.conf b/integration-tests/docker/router.conf new file mode 100644 index 00000000000..ddd8121f6c9 --- /dev/null +++ b/integration-tests/docker/router.conf @@ -0,0 +1,20 @@ +[program:druid-router] +command=java + -server + -Xmx1g + -XX:+UseConcMarkSweepGC + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOST_IP)s + -Ddruid.zk.service.host=druid-zookeeper + -Ddruid.computation.buffer.size=75000000 + -Ddruid.server.http.numThreads=100 + -Ddruid.processing.numThreads=1 + -cp /usr/local/druid/lib/* + io.druid.cli.Main server router +redirect_stderr=true +priority=100 +autorestart=false +stdout_logfile=/shared/logs/router.log diff --git a/integration-tests/docker/sample-data.sql b/integration-tests/docker/sample-data.sql new file mode 100644 index 00000000000..8221ced23e1 --- /dev/null +++ b/integration-tests/docker/sample-data.sql @@ -0,0 +1,5 @@ +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload) VALUES ('twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9','twitterstream','2013-05-13T01:08:18.192Z','2013-01-01T00:00:00.000Z','2013-01-02T00:00:00.000Z',0,'2013-01-02T04:13:41.980Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-01T00:00:00.000Z/2013-01-02T00:00:00.000Z\",\"version\":\"2013-01-02T04:13:41.980Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/2013-01-02T04:13:41.980Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":445235220,\"identifier\":\"twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9\"}'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload) VALUES ('twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9','twitterstream','2013-05-13T00:03:28.640Z','2013-01-02T00:00:00.000Z','2013-01-03T00:00:00.000Z',0,'2013-01-03T03:44:58.791Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-02T00:00:00.000Z/2013-01-03T00:00:00.000Z\",\"version\":\"2013-01-03T03:44:58.791Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z/2013-01-03T03:44:58.791Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":435325540,\"identifier\":\"twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9\"}'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload) VALUES ('twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9','twitterstream','2013-05-13T00:03:48.807Z','2013-01-03T00:00:00.000Z','2013-01-04T00:00:00.000Z',0,'2013-01-04T04:09:13.590Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-03T00:00:00.000Z/2013-01-04T00:00:00.000Z\",\"version\":\"2013-01-04T04:09:13.590Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z/2013-01-04T04:09:13.590Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":411651320,\"identifier\":\"twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9\"}'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload) VALUES ('wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9','wikipedia_editstream','2013-03-15T20:49:52.348Z','2012-12-29T00:00:00.000Z','2013-01-10T08:00:00.000Z',0,'2013-01-10T08:13:47.830Z_v9',1,'{\"dataSource\":\"wikipedia_editstream\",\"interval\":\"2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z\",\"version\":\"2013-01-10T08:13:47.830Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia_editstream/2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z/2013-01-10T08:13:47.830Z_v9/0/index.zip\"},\"dimensions\":\"anonymous,area_code,city,continent_code,country_name,dma_code,geo,language,namespace,network,newpage,page,postal_code,region_lookup,robot,unpatrolled,user\",\"metrics\":\"added,count,deleted,delta,delta_hist,unique_users,variation\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":446027801,\"identifier\":\"wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9\"}'); +INSERT INTO druid_segments (id, dataSource, created_date, start, end, partitioned, version, used, payload) VALUES ('wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', 'wikipedia', '2013-08-08T21:26:23.799Z', '2013-08-01T00:00:00.000Z', '2013-08-02T00:00:00.000Z', '0', '2013-08-08T21:22:48.989Z', '1', '{\"dataSource\":\"wikipedia\",\"interval\":\"2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z\",\"version\":\"2013-08-08T21:22:48.989Z\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip\"},\"dimensions\":\"dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace,anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup\",\"metrics\":\"count,delta,variation,added,deleted\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":24664730,\"identifier\":\"wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z\"}'); diff --git a/integration-tests/docker/supervisord.conf b/integration-tests/docker/supervisord.conf new file mode 100644 index 00000000000..99ab6b4b5cb --- /dev/null +++ b/integration-tests/docker/supervisord.conf @@ -0,0 +1,6 @@ +[supervisord] +nodaemon=true + +[include] +files = /usr/lib/druid/conf/*.conf + diff --git a/integration-tests/docker/zookeeper.conf b/integration-tests/docker/zookeeper.conf new file mode 100644 index 00000000000..1ee5247817f --- /dev/null +++ b/integration-tests/docker/zookeeper.conf @@ -0,0 +1,5 @@ +[program:zookeeper] +command=/usr/local/zookeeper/bin/zkServer.sh start-foreground +user=daemon +priority=0 +stdout_logfile=/shared/logs/zookeeper.log diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml new file mode 100644 index 00000000000..50c5e1f2ed3 --- /dev/null +++ b/integration-tests/pom.xml @@ -0,0 +1,151 @@ + + + + + 4.0.0 + io.druid + druid-integration-tests + druid-integration-tests + druid-integration-tests + + io.druid + druid + 0.7.0-SNAPSHOT + + + + + io.druid + druid-common + ${project.parent.version} + + + io.druid.extensions + druid-s3-extensions + ${project.parent.version} + + + io.druid.extensions + druid-histogram + ${project.parent.version} + + + io.druid.extensions + mysql-metadata-storage + ${project.parent.version} + + + io.druid + druid-services + ${project.parent.version} + + + io.druid + druid-server + ${project.parent.version} + + + + + org.testng + testng + + + org.easymock + easymock + test + + + + + + + maven-surefire-plugin + + + **/IT*.java + + + UTC + + + + + + + + integration-tests + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + build-and-start-druid-cluster + + exec + + pre-integration-test + + ${project.basedir}/run_cluster.sh + + + + stop-druid-cluster + + exec + + post-integration-test + + ${project.basedir}/stop_cluster.sh + + + + + + maven-failsafe-plugin + + + integration-tests + integration-test + + integration-test + verify + + + + + -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dtestrunfactory=org.testng.DruidTestRunnerFactory + -Ddruid.test.config.dockerIp=${env.DOCKER_IP} -Ddruid.zk.service.host=${env.DOCKER_IP} + + + src/test/resources/testng.xml + + + + + + + + + diff --git a/integration-tests/run_cluster.sh b/integration-tests/run_cluster.sh new file mode 100755 index 00000000000..eca4e65e60e --- /dev/null +++ b/integration-tests/run_cluster.sh @@ -0,0 +1,49 @@ +# cleanup +for node in druid-historical druid-coordinator druid-overlord druid-router druid-broker druid-middlemanager druid-zookeeper druid-metadata-storage; +do +docker stop $node +docker rm $node +done + +# environment variables +DIR=$(cd $(dirname $0) && pwd) +DOCKERDIR=$DIR/docker +SHARED_DIR=${HOME}/shared +SUPERVISORDIR=/usr/lib/druid/conf +RESOURCEDIR=$DIR/src/test/resources + +# Make directories if they dont exist +mkdir -p $SHARED_DIR/logs +mkdir -p $SHARED_DIR/tasklogs + +# install druid jars +rm -rf $SHARED_DIR/docker +cp -R docker $SHARED_DIR/docker +mvn dependency:copy-dependencies -DoutputDirectory=$SHARED_DIR/docker/lib + +# Build Druid Cluster Image +docker build -t druid/cluster $SHARED_DIR/docker + +# Start zookeeper +docker run -d --name druid-zookeeper -p 2181:2181 -v $SHARED_DIR:/shared -v $DOCKERDIR/zookeeper.conf:$SUPERVISORDIR/zookeeper.conf druid/cluster + +# Start MYSQL +docker run -d --name druid-metadata-storage -v $SHARED_DIR:/shared -v $DOCKERDIR/metadata-storage.conf:$SUPERVISORDIR/metadata-storage.conf druid/cluster + +# Start Overlord +docker run -d --name druid-overlord -p 8090:8090 -v $SHARED_DIR:/shared -v $DOCKERDIR/overlord.conf:$SUPERVISORDIR/overlord.conf --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper:druid-zookeeper druid/cluster + +# Start Coordinator +docker run -d --name druid-coordinator -p 8081:8081 -v $SHARED_DIR:/shared -v $DOCKERDIR/coordinator.conf:$SUPERVISORDIR/coordinator.conf --link druid-overlord:druid-overlord --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper:druid-zookeeper druid/cluster + +# Start Historical +docker run -d --name druid-historical -v $SHARED_DIR:/shared -v $DOCKERDIR/historical.conf:$SUPERVISORDIR/historical.conf --link druid-zookeeper:druid-zookeeper druid/cluster + +# Start Middlemanger +docker run -d --name druid-middlemanager -p 8100:8100 -p 8101:8101 -p 8102:8102 -p 8103:8103 -p 8104:8104 -p 8105:8105 -v $RESOURCEDIR:/resources -v $SHARED_DIR:/shared -v $DOCKERDIR/middlemanager.conf:$SUPERVISORDIR/middlemanager.conf --link druid-zookeeper:druid-zookeeper --link druid-overlord:druid-overlord druid/cluster + +# Start Broker +docker run -d --name druid-broker -v $SHARED_DIR:/shared -v $DOCKERDIR/broker.conf:$SUPERVISORDIR/broker.conf --link druid-zookeeper:druid-zookeeper --link druid-middlemanager:druid-middlemanager --link druid-historical:druid-historical druid/cluster + +# Start Router +docker run -d --name druid-router -p 8888:8888 -v $SHARED_DIR:/shared -v $DOCKERDIR/router.conf:$SUPERVISORDIR/router.conf --link druid-zookeeper:druid-zookeeper --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster diff --git a/integration-tests/src/main/java/io/druid/testing/DockerConfigProvider.java b/integration-tests/src/main/java/io/druid/testing/DockerConfigProvider.java new file mode 100644 index 00000000000..a11e9b082ff --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/DockerConfigProvider.java @@ -0,0 +1,67 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing; + + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.api.client.repackaged.com.google.common.base.Throwables; +import org.apache.commons.io.IOUtils; + +import javax.validation.constraints.NotNull; +import java.util.List; + +public class DockerConfigProvider implements IntegrationTestingConfigProvider +{ + + @JsonProperty + @NotNull + private String dockerIp; + + @Override + public IntegrationTestingConfig get() + { + return new IntegrationTestingConfig() + { + @Override + public String getCoordinatorHost() + { + return dockerIp+":8081"; + } + + @Override + public String getIndexerHost() + { + return dockerIp+":8090"; + } + + @Override + public String getRouterHost() + { + return dockerIp+ ":8888"; + } + + @Override + public String getMiddleManagerHost() + { + return dockerIp; + } + }; + } +} diff --git a/integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfig.java b/integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfig.java new file mode 100644 index 00000000000..dc94fb8ea1b --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfig.java @@ -0,0 +1,33 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing; + +/** + */ +public interface IntegrationTestingConfig +{ + public String getCoordinatorHost(); + + public String getIndexerHost(); + + public String getRouterHost(); + + public String getMiddleManagerHost(); +} diff --git a/integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfigProvider.java b/integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfigProvider.java new file mode 100644 index 00000000000..7de9d8fce7d --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfigProvider.java @@ -0,0 +1,32 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = DockerConfigProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "docker", value = DockerConfigProvider.class) +}) +public interface IntegrationTestingConfigProvider extends Provider +{ +} diff --git a/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java new file mode 100644 index 00000000000..cc0ac53f35a --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java @@ -0,0 +1,146 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.clients; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.metamx.common.ISE; +import com.metamx.common.logger.Logger; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.RequestBuilder; +import com.metamx.http.client.response.StatusResponseHandler; +import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.guice.annotations.Global; +import io.druid.testing.IntegrationTestingConfig; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.joda.time.Interval; + +import java.net.URL; +import java.net.URLEncoder; +import java.util.Map; + +public class CoordinatorResourceTestClient +{ + private final static Logger LOG = new Logger(CoordinatorResourceTestClient.class); + private final ObjectMapper jsonMapper; + private final HttpClient httpClient; + private final String coordinator; + private final StatusResponseHandler responseHandler; + + @Inject + CoordinatorResourceTestClient( + ObjectMapper jsonMapper, + @Global HttpClient httpClient, IntegrationTestingConfig config + ) + { + this.jsonMapper = jsonMapper; + this.httpClient = httpClient; + this.coordinator = config.getCoordinatorHost(); + this.responseHandler = new StatusResponseHandler(Charsets.UTF_8); + } + + private String getCoordinatorURL() + { + return String.format( + "http://%s/druid/coordinator/v1/", + coordinator + ); + } + + private Map getLoadStatus() + { + Map status = null; + try { + StatusResponseHolder response = makeRequest(HttpMethod.GET, getCoordinatorURL() + "loadstatus?simple"); + + status = jsonMapper.readValue( + response.getContent(), new TypeReference>() + { + } + ); + } + catch (Exception e) { + Throwables.propagate(e); + } + return status; + } + + public boolean areSegmentsLoaded(String dataSource) + { + final Map status = getLoadStatus(); + return (status.containsKey(dataSource) && status.get(dataSource) == 0); + } + + public void unloadSegmentsForDataSource(String dataSource, Interval interval) + { + killDataSource(dataSource, false, interval); + } + + public void deleteSegmentsDataSource(String dataSource, Interval interval) + { + killDataSource(dataSource, true, interval); + } + + private void killDataSource(String dataSource, boolean kill, Interval interval) + { + try { + makeRequest( + HttpMethod.DELETE, + String.format( + "%sdatasources/%s?kill=%s&interval=%s", + getCoordinatorURL(), + dataSource, kill, URLEncoder.encode(interval.toString(), "UTF-8") + ) + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + private StatusResponseHolder makeRequest(HttpMethod method, String url) + { + try { + StatusResponseHolder response = new RequestBuilder( + this.httpClient, + method, new URL(url) + ) + .go(responseHandler) + .get(); + if (!response.getStatus().equals(HttpResponseStatus.OK)) { + throw new ISE( + "Error while making request to url[%s] status[%s] content[%s]", + url, + response.getStatus(), + response.getContent() + ); + } + return response; + } + catch (Exception e) { + LOG.error(e, "Exception while sending request"); + throw Throwables.propagate(e); + } + } +} diff --git a/integration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java new file mode 100644 index 00000000000..ee212cdf112 --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java @@ -0,0 +1,135 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.clients; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.client.util.Charsets; +import com.google.common.base.Throwables; +import com.metamx.common.ISE; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.response.StatusResponseHandler; +import com.metamx.http.client.response.StatusResponseHolder; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +import javax.ws.rs.core.MediaType; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +public class EventReceiverFirehoseTestClient +{ + private final String host; + private final StatusResponseHandler responseHandler; + private final ObjectMapper jsonMapper; + private final HttpClient httpClient; + private final String chatID; + + public EventReceiverFirehoseTestClient(String host, String chatID, ObjectMapper jsonMapper, HttpClient httpClient) + { + this.host = host; + this.jsonMapper = jsonMapper; + this.responseHandler = new StatusResponseHandler(Charsets.UTF_8); + this.httpClient = httpClient; + this.chatID = chatID; + } + + private String getURL() + { + return String.format( + "http://%s/druid/worker/v1/chat/%s/push-events/", + host, + chatID + ); + } + + /** + * post events from the collection and return the count of events accepted + * + * @param events Collection of events to be posted + * + * @return + */ + public int postEvents(Collection> events) + { + try { + StatusResponseHolder response = httpClient.post(new URL(getURL())) + .setContent( + MediaType.APPLICATION_JSON, + this.jsonMapper.writeValueAsBytes(events) + ) + .go(responseHandler) + .get(); + if (!response.getStatus().equals(HttpResponseStatus.OK)) { + throw new ISE( + "Error while posting events to url[%s] status[%s] content[%s]", + getURL(), + response.getStatus(), + response.getContent() + ); + } + Map responseData = jsonMapper.readValue( + response.getContent(), new TypeReference>() + { + } + ); + return responseData.get("eventCount"); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + public int postEventsFromFile(String file) + { + try { + BufferedReader reader = new BufferedReader( + new InputStreamReader( + EventReceiverFirehoseTestClient.class.getResourceAsStream( + file + ) + ) + ); + String s; + Collection> events = new ArrayList>(); + while ((s = reader.readLine()) != null) { + events.add( + (Map) this.jsonMapper.readValue( + s, new TypeReference>() + { + } + ) + ); + } + int eventsPosted = postEvents(events); + if (eventsPosted != events.size()) { + throw new ISE("All events not posted, expected : %d actual : %d", events.size(), eventsPosted); + } + return eventsPosted; + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + } +} diff --git a/integration-tests/src/main/java/io/druid/testing/clients/OverlordResourceTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/OverlordResourceTestClient.java new file mode 100644 index 00000000000..af97b8b0bcf --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/clients/OverlordResourceTestClient.java @@ -0,0 +1,213 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.clients; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.metamx.common.ISE; +import com.metamx.common.logger.Logger; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.response.StatusResponseHandler; +import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.guice.annotations.Global; +import io.druid.indexing.common.TaskStatus; +import io.druid.indexing.common.task.Task; +import io.druid.testing.IntegrationTestingConfig; +import io.druid.testing.utils.RetryUtil; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +import java.net.URL; +import java.net.URLEncoder; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; + +public class OverlordResourceTestClient +{ + private final static Logger LOG = new Logger(OverlordResourceTestClient.class); + private final ObjectMapper jsonMapper; + private final HttpClient httpClient; + private final String indexer; + private final StatusResponseHandler responseHandler; + + @Inject + OverlordResourceTestClient( + ObjectMapper jsonMapper, + @Global HttpClient httpClient, IntegrationTestingConfig config + ) + { + this.jsonMapper = jsonMapper; + this.httpClient = httpClient; + this.indexer = config.getIndexerHost(); + this.responseHandler = new StatusResponseHandler(Charsets.UTF_8); + } + + private String getIndexerURL() + { + return String.format( + "http://%s/druid/indexer/v1/", + indexer + ); + } + + public String submitTask(Task task) + { + try { + return submitTask(this.jsonMapper.writeValueAsString(task)); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + public String submitTask(String task) + { + try { + StatusResponseHolder response = httpClient.post(new URL(getIndexerURL() + "task")) + .setContent( + "application/json", + task.getBytes() + ) + .go(responseHandler) + .get(); + if (!response.getStatus().equals(HttpResponseStatus.OK)) { + throw new ISE( + "Error while submitting task to indexer response [%s %s]", + response.getStatus(), + response.getContent() + ); + } + Map responseData = jsonMapper.readValue( + response.getContent(), new TypeReference>() + { + } + ); + String taskID = responseData.get("task"); + LOG.info("Submitted task with TaskID[%s]", taskID); + return taskID; + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + public TaskStatus.Status getTaskStatus(String taskID) + { + try { + StatusResponseHolder response = makeRequest( + String.format( + "%stask/%s/status", + getIndexerURL(), + URLEncoder.encode(taskID, "UTF-8") + ) + ); + + LOG.info("Index status response" + response.getContent()); + Map responseData = jsonMapper.readValue( + response.getContent(), new TypeReference>() + { + } + ); + //TODO: figure out a better way to parse the response... + String status = (String) ((Map) responseData.get("status")).get("status"); + return TaskStatus.Status.valueOf(status); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + public List getRunningTasks() + { + return getTasks("runningTasks"); + } + + public List getWaitingTasks() + { + return getTasks("waitingTasks"); + } + + public List getPendingTasks() + { + return getTasks("pendingTasks"); + } + + private List getTasks(String identifier) + { + try { + StatusResponseHolder response = makeRequest( + String.format("%s%s", getIndexerURL(), identifier) + ); + LOG.info("Tasks %s response %s", identifier, response.getContent()); + return jsonMapper.readValue( + response.getContent(), new TypeReference>() + { + } + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + public void waitUntilTaskCompletes(final String taskID) + { + RetryUtil.retryUntil( + new Callable() + { + @Override + public Boolean call() throws Exception + { + TaskStatus.Status status = getTaskStatus(taskID); + if (status == TaskStatus.Status.FAILED) { + throw new ISE("Indexer task FAILED"); + } + return status == TaskStatus.Status.SUCCESS; + } + }, + true, + 60000, + 10, + "Index Task to complete" + ); + } + + private StatusResponseHolder makeRequest(String url) + { + try { + StatusResponseHolder response = this.httpClient + .get(new URL(url)) + .go(responseHandler) + .get(); + if (!response.getStatus().equals(HttpResponseStatus.OK)) { + throw new ISE("Error while making request to indexer [%s %s]", response.getStatus(), response.getContent()); + } + return response; + } + catch (Exception e) { + LOG.error(e, "Exception while sending request"); + throw Throwables.propagate(e); + } + } + +} diff --git a/integration-tests/src/main/java/io/druid/testing/clients/QueryResourceTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/QueryResourceTestClient.java new file mode 100644 index 00000000000..4ba1d6ca7a5 --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/clients/QueryResourceTestClient.java @@ -0,0 +1,98 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.clients; + + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.metamx.common.ISE; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.response.StatusResponseHandler; +import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.guice.annotations.Global; +import io.druid.query.Query; +import io.druid.testing.IntegrationTestingConfig; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +import java.net.URL; +import java.util.List; +import java.util.Map; + +public class QueryResourceTestClient +{ + private final ObjectMapper jsonMapper; + private final HttpClient httpClient; + private final String router; + private final StatusResponseHandler responseHandler; + + @Inject + QueryResourceTestClient( + ObjectMapper jsonMapper, + @Global HttpClient httpClient, + IntegrationTestingConfig config + ) + { + this.jsonMapper = jsonMapper; + this.httpClient = httpClient; + this.router = config.getRouterHost(); + this.responseHandler = new StatusResponseHandler(Charsets.UTF_8); + } + + private String getBrokerURL() + { + return String.format( + "http://%s/druid/v2/", + router + ); + } + + public List> query(Query query) + { + try { + StatusResponseHolder response = httpClient.post(new URL(getBrokerURL())) + .setContent( + "application/json", + jsonMapper.writeValueAsBytes(query) + ) + .go(responseHandler) + .get(); + if (!response.getStatus().equals(HttpResponseStatus.OK)) { + throw new ISE( + "Error while querying[%s] status[%s] content[%s]", + getBrokerURL(), + response.getStatus(), + response.getContent() + ); + } + + return jsonMapper.readValue( + response.getContent(), new TypeReference>>() + { + } + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } +} diff --git a/integration-tests/src/main/java/io/druid/testing/clients/TaskResponseObject.java b/integration-tests/src/main/java/io/druid/testing/clients/TaskResponseObject.java new file mode 100644 index 00000000000..8b138505a6e --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/clients/TaskResponseObject.java @@ -0,0 +1,68 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.clients; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.indexing.common.TaskStatus; +import org.joda.time.DateTime; + +public class TaskResponseObject +{ + + private final String id; + private final DateTime createdTime; + private final DateTime queueInsertionTime; + private final TaskStatus status; + + @JsonCreator + private TaskResponseObject( + @JsonProperty("id") String id, + @JsonProperty("createdTime") DateTime createdTime, + @JsonProperty("queueInsertionTime") DateTime queueInsertionTime, + @JsonProperty("status") TaskStatus status + ) + { + this.id = id; + this.createdTime = createdTime; + this.queueInsertionTime = queueInsertionTime; + this.status = status; + } + + public String getId() + { + return id; + } + + public DateTime getCreatedTime() + { + return createdTime; + } + + public DateTime getQueueInsertionTime() + { + return queueInsertionTime; + } + + public TaskStatus getStatus() + { + return status; + } +} diff --git a/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModule.java b/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModule.java new file mode 100644 index 00000000000..95dd35d3274 --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModule.java @@ -0,0 +1,54 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.guice; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.metamx.emitter.core.LoggingEmitter; +import com.metamx.emitter.core.LoggingEmitterConfig; +import com.metamx.emitter.service.ServiceEmitter; +import io.druid.guice.ConfigProvider; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.guice.ManageLifecycle; +import io.druid.testing.IntegrationTestingConfig; +import io.druid.testing.IntegrationTestingConfigProvider; + +/** + */ +public class DruidTestModule implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(IntegrationTestingConfig.class).toProvider(IntegrationTestingConfigProvider.class).in(ManageLifecycle.class); + JsonConfigProvider.bind(binder, "druid.test.config", IntegrationTestingConfigProvider.class); + } + + @Provides + @LazySingleton + public ServiceEmitter getServiceEmitter(Supplier config, ObjectMapper jsonMapper) + { + return new ServiceEmitter("", "", new LoggingEmitter(config.get(), jsonMapper)); + } +} diff --git a/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModuleFactory.java b/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModuleFactory.java new file mode 100644 index 00000000000..062a87c130f --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModuleFactory.java @@ -0,0 +1,63 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.guice; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Injector; +import com.google.inject.Module; +import io.druid.guice.GuiceInjectors; +import io.druid.guice.IndexingServiceFirehoseModule; +import io.druid.initialization.Initialization; +import org.testng.IModuleFactory; +import org.testng.ITestContext; + +import java.util.Collections; +import java.util.List; + +public class DruidTestModuleFactory implements IModuleFactory +{ + private static final Module module = new DruidTestModule(); + private static final Injector injector = Initialization.makeInjectorWithModules( + GuiceInjectors.makeStartupInjector(), + getModules() + ); + + public static Injector getInjector() + { + return injector; + } + + private static List getModules() + { + return ImmutableList.of( + new DruidTestModule(), + new IndexingServiceFirehoseModule() + ); + } + + @Override + public Module createModule(ITestContext context, Class testClass) + { + context.addGuiceModule(DruidTestModule.class, module); + context.addInjector(Collections.singletonList(module), injector); + return module; + } + +} diff --git a/integration-tests/src/main/java/io/druid/testing/utils/FromFileTestQueryHelper.java b/integration-tests/src/main/java/io/druid/testing/utils/FromFileTestQueryHelper.java new file mode 100644 index 00000000000..4c815d68843 --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/utils/FromFileTestQueryHelper.java @@ -0,0 +1,79 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.utils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.metamx.common.ISE; +import com.metamx.common.logger.Logger; +import io.druid.testing.clients.QueryResourceTestClient; + +import java.util.List; +import java.util.Map; + +public class FromFileTestQueryHelper +{ + public static Logger LOG = new Logger(FromFileTestQueryHelper.class); + private final QueryResourceTestClient queryClient; + private final ObjectMapper jsonMapper; + + @Inject + FromFileTestQueryHelper(ObjectMapper jsonMapper, QueryResourceTestClient queryClient) + { + this.jsonMapper = jsonMapper; + this.queryClient = queryClient; + } + + public void testQueriesFromFile(String filePath, int timesToRun) throws Exception + { + LOG.info("Starting query tests for [%s]", filePath); + List queries = + jsonMapper.readValue( + FromFileTestQueryHelper.class.getResourceAsStream(filePath), + new TypeReference>() + { + } + ); + for (int i = 0; i < timesToRun; i++) { + LOG.info("Starting Iteration " + i); + + boolean failed = false; + for (QueryWithResults queryWithResult : queries) { + LOG.info("Running Query " + queryWithResult.getQuery().getType()); + List> result = queryClient.query(queryWithResult.getQuery()); + if (!QueryResultVerifier.compareResults(result, queryWithResult.getExpectedResults())) { + LOG.error( + "Failed while executing %s actualResults : %s", + queryWithResult, + jsonMapper.writeValueAsString(result) + ); + failed = true; + } else { + LOG.info("Results Verified for Query " + queryWithResult.getQuery().getType()); + } + } + + if (failed) { + throw new ISE("one or more twitter queries failed"); + } + } + } +} diff --git a/integration-tests/src/main/java/io/druid/testing/utils/QueryResultVerifier.java b/integration-tests/src/main/java/io/druid/testing/utils/QueryResultVerifier.java new file mode 100644 index 00000000000..fee833f5b29 --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/utils/QueryResultVerifier.java @@ -0,0 +1,49 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.utils; + +import java.util.Iterator; +import java.util.Map; + +public class QueryResultVerifier +{ + public static boolean compareResults( + Iterable> actual, + Iterable> expected + ) + { + Iterator> actualIter = actual.iterator(); + Iterator> expectedIter = expected.iterator(); + + while (actualIter.hasNext() && expectedIter.hasNext()) { + Map actualRes = actualIter.next(); + Map expRes = expectedIter.next(); + + if (!actualRes.equals(expRes)) { + return false; + } + } + + if (actualIter.hasNext() || expectedIter.hasNext()) { + return false; + } + return true; + } +} diff --git a/integration-tests/src/main/java/io/druid/testing/utils/QueryWithResults.java b/integration-tests/src/main/java/io/druid/testing/utils/QueryWithResults.java new file mode 100644 index 00000000000..2ad56d3263b --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/utils/QueryWithResults.java @@ -0,0 +1,64 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.utils; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.query.Query; + +import java.util.List; +import java.util.Map; + +public class QueryWithResults +{ + private final Query query; + private final List> expectedResults; + + @JsonCreator + public QueryWithResults( + @JsonProperty("query") Query query, + @JsonProperty("expectedResults") List> expectedResults + ) + { + this.query = query; + this.expectedResults = expectedResults; + } + + @JsonProperty + public Query getQuery() + { + return query; + } + + @JsonProperty + public List> getExpectedResults() + { + return expectedResults; + } + + @Override + public String toString() + { + return "QueryWithResults{" + + "query=" + query + + ", expectedResults=" + expectedResults + + '}'; + } +} \ No newline at end of file diff --git a/integration-tests/src/main/java/io/druid/testing/utils/RetryUtil.java b/integration-tests/src/main/java/io/druid/testing/utils/RetryUtil.java new file mode 100644 index 00000000000..0c8451bf039 --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/utils/RetryUtil.java @@ -0,0 +1,76 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.utils; + +import com.google.common.base.Throwables; +import com.metamx.common.ISE; +import com.metamx.common.logger.Logger; + +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +public class RetryUtil +{ + + private static final Logger LOG = new Logger(RetryUtil.class); + + public static int DEFAULT_RETRY_COUNT = 10; + + public static long DEFAULT_RETRY_SLEEP = TimeUnit.SECONDS.toMillis(30); + + public static void retryUntilTrue(Callable callable, String task) + { + retryUntil(callable, true, DEFAULT_RETRY_SLEEP, DEFAULT_RETRY_COUNT, task); + } + + public static void retryUntilFalse(Callable callable, String task) + { + retryUntil(callable, false, DEFAULT_RETRY_SLEEP, DEFAULT_RETRY_COUNT, task); + } + + public static void retryUntil( + Callable callable, + boolean expectedValue, + long delayInMillis, + int retryCount, + String taskMessage + ) + { + try { + int currentTry = 0; + while (callable.call() != expectedValue) { + if (currentTry > retryCount) { + throw new ISE("Max number of retries[%d] exceeded for Task[%s]. Failing.", retryCount, taskMessage); + } + LOG.info( + "Attempt[%d]: Task %s still not complete. Next retry in %d ms", + currentTry, taskMessage, delayInMillis + ); + Thread.sleep(delayInMillis); + + currentTry++; + } + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + +} diff --git a/integration-tests/src/main/java/io/druid/testing/utils/ServerDiscoveryUtil.java b/integration-tests/src/main/java/io/druid/testing/utils/ServerDiscoveryUtil.java new file mode 100644 index 00000000000..e675a2a35e9 --- /dev/null +++ b/integration-tests/src/main/java/io/druid/testing/utils/ServerDiscoveryUtil.java @@ -0,0 +1,64 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.testing.utils; + +import com.metamx.common.logger.Logger; +import io.druid.client.selector.Server; +import io.druid.curator.discovery.ServerDiscoverySelector; + +import java.util.concurrent.Callable; + +public class ServerDiscoveryUtil +{ + + private static final Logger LOG = new Logger(ServerDiscoveryUtil.class); + + public static boolean isInstanceReady(ServerDiscoverySelector serviceProvider) + { + try { + Server instance = serviceProvider.pick(); + if (instance == null) { + LOG.warn("Unable to find a host"); + return false; + } + } + catch (Exception e) { + LOG.error(e, "Caught exception waiting for host"); + return false; + } + return true; + } + + public static void waitUntilInstanceReady(final ServerDiscoverySelector serviceProvider, String instanceType) + { + RetryUtil.retryUntilTrue( + new Callable() + { + @Override + public Boolean call() throws Exception + { + return isInstanceReady(serviceProvider); + } + }, + String.format("Instance %s to get ready", instanceType) + ); + } + +} diff --git a/integration-tests/src/main/java/org/testng/DruidTestRunnerFactory.java b/integration-tests/src/main/java/org/testng/DruidTestRunnerFactory.java new file mode 100644 index 00000000000..6d0f3684938 --- /dev/null +++ b/integration-tests/src/main/java/org/testng/DruidTestRunnerFactory.java @@ -0,0 +1,150 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.testng; + +import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.api.client.util.Charsets; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.common.logger.Logger; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.response.StatusResponseHandler; +import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.guice.annotations.Global; +import io.druid.testing.IntegrationTestingConfig; +import io.druid.testing.guice.DruidTestModuleFactory; +import io.druid.testing.utils.RetryUtil; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.testng.internal.IConfiguration; +import org.testng.internal.annotations.IAnnotationFinder; +import org.testng.xml.XmlTest; + +import java.net.URL; +import java.util.List; +import java.util.concurrent.Callable; + +public class DruidTestRunnerFactory implements ITestRunnerFactory +{ + private static final Logger LOG = new Logger(DruidTestRunnerFactory.class); + + @Override + public TestRunner newTestRunner( + ISuite suite, XmlTest test, List listeners + ) + { + IConfiguration configuration = TestNG.getDefault().getConfiguration(); + String outputDirectory = suite.getOutputDirectory(); + IAnnotationFinder annotationFinder = configuration.getAnnotationFinder(); + Boolean skipFailedInvocationCounts = suite.getXmlSuite().skipFailedInvocationCounts(); + return new DruidTestRunner( + configuration, + suite, + test, + outputDirectory, + annotationFinder, + skipFailedInvocationCounts, + listeners + ); + } + + private static class DruidTestRunner extends TestRunner + { + + protected DruidTestRunner( + IConfiguration configuration, + ISuite suite, + XmlTest test, + String outputDirectory, + IAnnotationFinder finder, + boolean skipFailedInvocationCounts, + List invokedMethodListeners + ) + { + super(configuration, suite, test, outputDirectory, finder, skipFailedInvocationCounts, invokedMethodListeners); + } + + @Override + public void run() + { + Injector injector = DruidTestModuleFactory.getInjector(); + IntegrationTestingConfig config = injector.getInstance(IntegrationTestingConfig.class); + HttpClient client = injector.getInstance(Key.get(HttpClient.class, Global.class)); + ; + waitUntilInstanceReady(client, config.getCoordinatorHost()); + waitUntilInstanceReady(client, config.getIndexerHost()); + waitUntilInstanceReady(client, config.getRouterHost()); + Lifecycle lifecycle = injector.getInstance(Lifecycle.class); + try { + lifecycle.start(); + runTests(); + } + catch (Exception e) { + e.printStackTrace(); + throw Throwables.propagate(e); + } + finally { + lifecycle.stop(); + } + + } + + private void runTests() + { + super.run(); + } + + public void waitUntilInstanceReady(final HttpClient client, final String host) + { + final StatusResponseHandler handler = new StatusResponseHandler(Charsets.UTF_8); + RetryUtil.retryUntilTrue( + new Callable() + { + @Override + public Boolean call() throws Exception + { + try { + StatusResponseHolder response = client.get( + new URL( + String.format( + "http://%s/status", + host + ) + ) + ) + .go(handler) + .get(); + System.out.println(response.getStatus() + response.getContent()); + if (response.getStatus().equals(HttpResponseStatus.OK)) { + return true; + } else { + return false; + } + } + catch (Throwable e) { + e.printStackTrace(); + return false; + } + } + }, "Waiting for instance to be ready: [" + host + "]" + ); + } + } +} diff --git a/integration-tests/src/main/java/org/testng/TestNG.java b/integration-tests/src/main/java/org/testng/TestNG.java new file mode 100644 index 00000000000..c02793e5486 --- /dev/null +++ b/integration-tests/src/main/java/org/testng/TestNG.java @@ -0,0 +1,2252 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.testng; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.ParameterException; +import org.testng.annotations.ITestAnnotation; +import org.testng.collections.Lists; +import org.testng.collections.Maps; +import org.testng.internal.ClassHelper; +import org.testng.internal.Configuration; +import org.testng.internal.DynamicGraph; +import org.testng.internal.IConfiguration; +import org.testng.internal.IResultListener2; +import org.testng.internal.OverrideProcessor; +import org.testng.internal.SuiteRunnerMap; +import org.testng.internal.Utils; +import org.testng.internal.Version; +import org.testng.internal.annotations.DefaultAnnotationTransformer; +import org.testng.internal.annotations.IAnnotationFinder; +import org.testng.internal.annotations.JDK15AnnotationFinder; +import org.testng.internal.annotations.Sets; +import org.testng.internal.thread.graph.GraphThreadPoolExecutor; +import org.testng.internal.thread.graph.IThreadWorkerFactory; +import org.testng.internal.thread.graph.SuiteWorkerFactory; +import org.testng.junit.JUnitTestFinder; +import org.testng.log4testng.Logger; +import org.testng.remote.SuiteDispatcher; +import org.testng.remote.SuiteSlave; +import org.testng.reporters.EmailableReporter; +import org.testng.reporters.EmailableReporter2; +import org.testng.reporters.FailedReporter; +import org.testng.reporters.JUnitReportReporter; +import org.testng.reporters.SuiteHTMLReporter; +import org.testng.reporters.VerboseReporter; +import org.testng.reporters.XMLReporter; +import org.testng.reporters.jq.Main; +import org.testng.xml.Parser; +import org.testng.xml.XmlClass; +import org.testng.xml.XmlInclude; +import org.testng.xml.XmlMethodSelector; +import org.testng.xml.XmlSuite; +import org.testng.xml.XmlTest; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +import static org.testng.internal.Utils.defaultIfStringEmpty; +import static org.testng.internal.Utils.isStringEmpty; +import static org.testng.internal.Utils.isStringNotEmpty; + +/** + * Class copied from TestNG library ver 6.8.7 to apply a workaround for http://jira.codehaus.org/browse/SUREFIRE-622 + * To Locate the PATCHED AREA search for keyword "PATCH" in this class file + *

+ *

+ * This class is the main entry point for running tests in the TestNG framework. + * Users can create their own TestNG object and invoke it in many different + * ways: + *

    + *
  • On an existing testng.xml + *
  • On a synthetic testng.xml, created entirely from Java + *
  • By directly setting the test classes + *
+ * You can also define which groups to include or exclude, assign parameters, etc... + *

+ * The command line parameters are: + *

    + *
  • -d outputdir: specify the output directory
    • + *
  • -testclass class_name: specifies one or several class names
    • + *
  • -testjar jar_name: specifies the jar containing the tests
    • + *
  • -sourcedir src1;src2: ; separated list of source directories + * (used only when javadoc annotations are used)
    • + *
  • -target
    • + *
  • -groups
    • + *
  • -testrunfactory
    • + *
  • -listener
    • + *
+ *

+ * Please consult documentation for more details. + *

+ * FIXME: should support more than simple paths for suite xmls + * + * @author Cedric Beust + * @author Alex Popescu + * @see #usage() + */ +public class TestNG +{ + + /** + * This class' log4testng Logger. + */ + private static final Logger LOGGER = Logger.getLogger(TestNG.class); + + /** + * The default name for a suite launched from the command line + */ + public static final String DEFAULT_COMMAND_LINE_SUITE_NAME = "Command line suite"; + + /** + * The default name for a test launched from the command line + */ + public static final String DEFAULT_COMMAND_LINE_TEST_NAME = "Command line test"; + + /** + * The default name of the result's output directory (keep public, used by Eclipse). + */ + public static final String DEFAULT_OUTPUTDIR = "test-output"; + + /** + * System properties + */ + public static final String SHOW_TESTNG_STACK_FRAMES = "testng.show.stack.frames"; + public static final String TEST_CLASSPATH = "testng.test.classpath"; + + private static TestNG m_instance; + + private static JCommander m_jCommander; + + private List m_commandLineMethods; + protected List m_suites = Lists.newArrayList(); + private List m_cmdlineSuites; + private String m_outputDir = DEFAULT_OUTPUTDIR; + + private String[] m_includedGroups; + private String[] m_excludedGroups; + + private Boolean m_isJUnit = XmlSuite.DEFAULT_JUNIT; + private Boolean m_isMixed = XmlSuite.DEFAULT_MIXED; + protected boolean m_useDefaultListeners = true; + + private ITestRunnerFactory m_testRunnerFactory; + + // These listeners can be overridden from the command line + private List m_testListeners = Lists.newArrayList(); + private List m_suiteListeners = Lists.newArrayList(); + private Set m_reporters = Sets.newHashSet(); + + protected static final int HAS_FAILURE = 1; + protected static final int HAS_SKIPPED = 2; + protected static final int HAS_FSP = 4; + protected static final int HAS_NO_TEST = 8; + + public static final Integer DEFAULT_VERBOSE = 1; + + private int m_status; + private boolean m_hasTests = false; + + private String m_slavefileName = null; + private String m_masterfileName = null; + + // Command line suite parameters + private int m_threadCount; + private boolean m_useThreadCount; + private String m_parallelMode; + private boolean m_useParallelMode; + private String m_configFailurePolicy; + private Class[] m_commandLineTestClasses; + + private String m_defaultSuiteName = DEFAULT_COMMAND_LINE_SUITE_NAME; + private String m_defaultTestName = DEFAULT_COMMAND_LINE_TEST_NAME; + + private Map m_methodDescriptors = Maps.newHashMap(); + + private ITestObjectFactory m_objectFactory; + + private List m_invokedMethodListeners = Lists.newArrayList(); + + private Integer m_dataProviderThreadCount = null; + + private String m_jarPath; + /** + * The path of the testng.xml file inside the jar file + */ + private String m_xmlPathInJar = CommandLineArgs.XML_PATH_IN_JAR_DEFAULT; + + private List m_stringSuites = Lists.newArrayList(); + + private IHookable m_hookable; + private IConfigurable m_configurable; + + protected long m_end; + protected long m_start; + + private List m_executionListeners = Lists.newArrayList(); + + private boolean m_isInitialized = false; + + /** + * Default constructor. Setting also usage of default listeners/reporters. + */ + public TestNG() + { + init(true); + } + + /** + * Used by maven2 to have 0 output of any kind come out + * of testng. + * + * @param useDefaultListeners Whether or not any default reports + * should be added to tests. + */ + public TestNG(boolean useDefaultListeners) + { + init(useDefaultListeners); + } + + private void init(boolean useDefaultListeners) + { + m_instance = this; + + m_useDefaultListeners = useDefaultListeners; + m_configuration = new Configuration(); + } + + public int getStatus() + { + return m_status; + } + + private void setStatus(int status) + { + m_status |= status; + } + + /** + * Sets the output directory where the reports will be created. + * + * @param outputdir The directory. + */ + public void setOutputDirectory(final String outputdir) + { + if (isStringNotEmpty(outputdir)) { + m_outputDir = outputdir; + } + } + + /** + * If this method is passed true before run(), the default listeners + * will not be used. + *

    + *
  • org.testng.reporters.TestHTMLReporter + *
  • org.testng.reporters.JUnitXMLReporter + *
  • org.testng.reporters.XMLReporter + *
+ * + * @see org.testng.reporters.TestHTMLReporter + * @see org.testng.reporters.JUnitXMLReporter + * @see org.testng.reporters.XMLReporter + */ + public void setUseDefaultListeners(boolean useDefaultListeners) + { + m_useDefaultListeners = useDefaultListeners; + } + + /** + * Sets a jar containing a testng.xml file. + * + * @param jarPath + */ + public void setTestJar(String jarPath) + { + m_jarPath = jarPath; + } + + /** + * Sets the path to the XML file in the test jar file. + */ + public void setXmlPathInJar(String xmlPathInJar) + { + m_xmlPathInJar = xmlPathInJar; + } + + public void initializeSuitesAndJarFile() + { + // The Eclipse plug-in (RemoteTestNG) might have invoked this method already + // so don't initialize suites twice. + if (m_isInitialized) { + return; + } + + m_isInitialized = true; + if (m_suites.size() > 0) { + //to parse the suite files (), if any + for (XmlSuite s : m_suites) { + + for (String suiteFile : s.getSuiteFiles()) { + try { + Collection childSuites = getParser(suiteFile).parse(); + for (XmlSuite cSuite : childSuites) { + cSuite.setParentSuite(s); + s.getChildSuites().add(cSuite); + } + } + catch (FileNotFoundException e) { + e.printStackTrace(System.out); + } + catch (ParserConfigurationException e) { + e.printStackTrace(System.out); + } + catch (SAXException e) { + e.printStackTrace(System.out); + } + catch (IOException e) { + e.printStackTrace(System.out); + } + } + + } + return; + } + + // + // Parse the suites that were passed on the command line + // + for (String suitePath : m_stringSuites) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("suiteXmlPath: \"" + suitePath + "\""); + } + try { + Collection allSuites = getParser(suitePath).parse(); + + for (XmlSuite s : allSuites) { + // If test names were specified, only run these test names + if (m_testNames != null) { + m_suites.add(extractTestNames(s, m_testNames)); + } else { + m_suites.add(s); + } + } + } + catch (FileNotFoundException e) { + e.printStackTrace(System.out); + } + catch (IOException e) { + e.printStackTrace(System.out); + } + catch (ParserConfigurationException e) { + e.printStackTrace(System.out); + } + catch (SAXException e) { + e.printStackTrace(System.out); + } + catch (Exception ex) { + // Probably a Yaml exception, unnest it + Throwable t = ex; + while (t.getCause() != null) { + t = t.getCause(); + } +// t.printStackTrace(); + if (t instanceof TestNGException) { + throw (TestNGException) t; + } else { + throw new TestNGException(t); + } + } + } + + // + // jar path + // + // If suites were passed on the command line, they take precedence over the suite file + // inside that jar path + if (m_jarPath != null && m_stringSuites.size() > 0) { + StringBuilder suites = new StringBuilder(); + for (String s : m_stringSuites) { + suites.append(s); + } + Utils.log( + "TestNG", 2, "Ignoring the XML file inside " + m_jarPath + " and using " + + suites + " instead" + ); + return; + } + if (isStringEmpty(m_jarPath)) { + return; + } + + // We have a jar file and no XML file was specified: try to find an XML file inside the jar + File jarFile = new File(m_jarPath); + + try { + + Utils.log("TestNG", 2, "Trying to open jar file:" + jarFile); + + JarFile jf = new JarFile(jarFile); +// System.out.println(" result: " + jf); + Enumeration entries = jf.entries(); + List classes = Lists.newArrayList(); + boolean foundTestngXml = false; + while (entries.hasMoreElements()) { + JarEntry je = entries.nextElement(); + if (je.getName().equals(m_xmlPathInJar)) { + Parser parser = getParser(jf.getInputStream(je)); + m_suites.addAll(parser.parse()); + foundTestngXml = true; + break; + } else if (je.getName().endsWith(".class")) { + int n = je.getName().length() - ".class".length(); + classes.add(je.getName().replace("/", ".").substring(0, n)); + } + } + if (!foundTestngXml) { + Utils.log( + "TestNG", 1, + "Couldn't find the " + m_xmlPathInJar + " in the jar file, running all the classes" + ); + XmlSuite xmlSuite = new XmlSuite(); + xmlSuite.setVerbose(0); + xmlSuite.setName("Jar suite"); + XmlTest xmlTest = new XmlTest(xmlSuite); + List xmlClasses = Lists.newArrayList(); + for (String cls : classes) { + XmlClass xmlClass = new XmlClass(cls); + xmlClasses.add(xmlClass); + } + xmlTest.setXmlClasses(xmlClasses); + m_suites.add(xmlSuite); + } + } + catch (ParserConfigurationException ex) { + ex.printStackTrace(); + } + catch (SAXException ex) { + ex.printStackTrace(); + } + catch (IOException ex) { + ex.printStackTrace(); + } + } + + private Parser getParser(String path) + { + Parser result = new Parser(path); + initProcessor(result); + return result; + } + + private Parser getParser(InputStream is) + { + Parser result = new Parser(is); + initProcessor(result); + return result; + } + + private void initProcessor(Parser result) + { + result.setPostProcessor(new OverrideProcessor(m_includedGroups, m_excludedGroups)); + } + + /** + * If the XmlSuite contains at least one test named as testNames, return + * an XmlSuite that's made only of these tests, otherwise, return the + * original suite. + */ + private static XmlSuite extractTestNames(XmlSuite s, List testNames) + { + List tests = Lists.newArrayList(); + for (XmlTest xt : s.getTests()) { + for (String tn : testNames) { + if (xt.getName().equals(tn)) { + tests.add(xt); + } + } + } + + if (tests.size() == 0) { + return s; + } else { + XmlSuite result = (XmlSuite) s.clone(); + result.getTests().clear(); + result.getTests().addAll(tests); + return result; + } + } + + /** + * Define the number of threads in the thread pool. + */ + public void setThreadCount(int threadCount) + { + if (threadCount < 1) { + exitWithError("Cannot use a threadCount parameter less than 1; 1 > " + threadCount); + } + + m_threadCount = threadCount; + m_useThreadCount = true; + } + + /** + * Define whether this run will be run in parallel mode. + */ + public void setParallel(String parallel) + { + m_parallelMode = parallel; + m_useParallelMode = true; + } + + public void setCommandLineSuite(XmlSuite suite) + { + m_cmdlineSuites = Lists.newArrayList(); + m_cmdlineSuites.add(suite); + m_suites.add(suite); + } + + /** + * Set the test classes to be run by this TestNG object. This method + * will create a dummy suite that will wrap these classes called + * "Command Line Test". + *

+ * If used together with threadCount, parallel, groups, excludedGroups than this one must be set first. + * + * @param classes An array of classes that contain TestNG annotations. + */ + public void setTestClasses(Class[] classes) + { + m_suites.clear(); + m_commandLineTestClasses = classes; + } + + /** + * Given a string com.example.Foo.f1, return an array where [0] is the class and [1] + * is the method. + */ + private String[] splitMethod(String m) + { + int index = m.lastIndexOf("."); + if (index < 0) { + throw new TestNGException( + "Bad format for command line method:" + m + + ", expected ." + ); + } + + return new String[]{m.substring(0, index), m.substring(index + 1).replaceAll("\\*", "\\.\\*")}; + } + + /** + * @param commandLineMethods a string with the form "com.example.Foo.f1,com.example.Bar.f2" + * + * @return a list of XmlSuite objects that represent the list of classes and methods passed + * in parameter. + */ + private List createCommandLineSuitesForMethods(List commandLineMethods) + { + // + // Create the tag + // + Set classes = Sets.newHashSet(); + for (String m : commandLineMethods) { + Class c = ClassHelper.forName(splitMethod(m)[0]); + if (c != null) { + classes.add(c); + } + } + + List result = createCommandLineSuitesForClasses(classes.toArray(new Class[0])); + + // + // Add the method tags + // + List xmlClasses = Lists.newArrayList(); + for (XmlSuite s : result) { + for (XmlTest t : s.getTests()) { + xmlClasses.addAll(t.getClasses()); + } + } + + for (XmlClass xc : xmlClasses) { + for (String m : commandLineMethods) { + String[] split = splitMethod(m); + String className = split[0]; + if (xc.getName().equals(className)) { + XmlInclude includedMethod = new XmlInclude(split[1]); + xc.getIncludedMethods().add(includedMethod); + } + } + } + + return result; + } + + private List createCommandLineSuitesForClasses(Class[] classes) + { + // + // See if any of the classes has an xmlSuite or xmlTest attribute. + // If it does, create the appropriate XmlSuite, otherwise, create + // the default one + // + XmlClass[] xmlClasses = Utils.classesToXmlClasses(classes); + Map suites = Maps.newHashMap(); + IAnnotationFinder finder = m_configuration.getAnnotationFinder(); + + for (int i = 0; i < classes.length; i++) { + Class c = classes[i]; + ITestAnnotation test = finder.findAnnotation(c, ITestAnnotation.class); + String suiteName = getDefaultSuiteName(); + String testName = getDefaultTestName(); + boolean isJUnit = false; + if (test != null) { + suiteName = defaultIfStringEmpty(test.getSuiteName(), suiteName); + testName = defaultIfStringEmpty(test.getTestName(), testName); + } else { + if (m_isMixed && JUnitTestFinder.isJUnitTest(c)) { + isJUnit = true; + testName = c.getName(); + } + } + XmlSuite xmlSuite = suites.get(suiteName); + if (xmlSuite == null) { + xmlSuite = new XmlSuite(); + xmlSuite.setName(suiteName); + suites.put(suiteName, xmlSuite); + } + + if (m_dataProviderThreadCount != null) { + xmlSuite.setDataProviderThreadCount(m_dataProviderThreadCount); + } + XmlTest xmlTest = null; + for (XmlTest xt : xmlSuite.getTests()) { + if (xt.getName().equals(testName)) { + xmlTest = xt; + break; + } + } + + if (xmlTest == null) { + xmlTest = new XmlTest(xmlSuite); + xmlTest.setName(testName); + xmlTest.setJUnit(isJUnit); + } + + xmlTest.getXmlClasses().add(xmlClasses[i]); + } + + return new ArrayList(suites.values()); + } + + public void addMethodSelector(String className, int priority) + { + m_methodDescriptors.put(className, priority); + } + + /** + * Set the suites file names to be run by this TestNG object. This method tries to load and + * parse the specified TestNG suite xml files. If a file is missing, it is ignored. + * + * @param suites A list of paths to one more XML files defining the tests. For example: + *

+ * 

+   *                             TestNG tng = new TestNG();
+   *                             List suites = Lists.newArrayList();
+   *                             suites.add("c:/tests/testng1.xml");
+   *                             suites.add("c:/tests/testng2.xml");
+   *                             tng.setTestSuites(suites);
+   *                             tng.run();
+   *
+ */ + public void setTestSuites(List suites) + { + m_stringSuites = suites; + } + + /** + * Specifies the XmlSuite objects to run. + * + * @param suites + * + * @see org.testng.xml.XmlSuite + */ + public void setXmlSuites(List suites) + { + m_suites = suites; + } + + /** + * Define which groups will be excluded from this run. + * + * @param groups A list of group names separated by a comma. + */ + public void setExcludedGroups(String groups) + { + m_excludedGroups = Utils.split(groups, ","); + } + + + /** + * Define which groups will be included from this run. + * + * @param groups A list of group names separated by a comma. + */ + public void setGroups(String groups) + { + m_includedGroups = Utils.split(groups, ","); + } + + + private void setTestRunnerFactoryClass(Class testRunnerFactoryClass) + { + setTestRunnerFactory((ITestRunnerFactory) ClassHelper.newInstance(testRunnerFactoryClass)); + } + + + protected void setTestRunnerFactory(ITestRunnerFactory itrf) + { + m_testRunnerFactory = itrf; + } + + public void setObjectFactory(Class c) + { + m_objectFactory = (ITestObjectFactory) ClassHelper.newInstance(c); + } + + public void setObjectFactory(ITestObjectFactory factory) + { + m_objectFactory = factory; + } + + /** + * Define which listeners to user for this run. + * + * @param classes A list of classes, which must be either ISuiteListener, + * ITestListener or IReporter + */ + public void setListenerClasses(List classes) + { + for (Class cls : classes) { + addListener(ClassHelper.newInstance(cls)); + } + } + + public void addListener(Object listener) + { + if (!(listener instanceof ITestNGListener)) { + exitWithError( + "Listener " + listener + + " must be one of ITestListener, ISuiteListener, IReporter, " + + " IAnnotationTransformer, IMethodInterceptor or IInvokedMethodListener" + ); + } else { + if (listener instanceof ISuiteListener) { + addListener((ISuiteListener) listener); + } + if (listener instanceof ITestListener) { + addListener((ITestListener) listener); + } + if (listener instanceof IReporter) { + addListener((IReporter) listener); + } + if (listener instanceof IAnnotationTransformer) { + setAnnotationTransformer((IAnnotationTransformer) listener); + } + if (listener instanceof IMethodInterceptor) { + setMethodInterceptor((IMethodInterceptor) listener); + } + if (listener instanceof IInvokedMethodListener) { + addInvokedMethodListener((IInvokedMethodListener) listener); + } + if (listener instanceof IHookable) { + setHookable((IHookable) listener); + } + if (listener instanceof IConfigurable) { + setConfigurable((IConfigurable) listener); + } + if (listener instanceof IExecutionListener) { + addExecutionListener((IExecutionListener) listener); + } + if (listener instanceof IConfigurationListener) { + getConfiguration().addConfigurationListener((IConfigurationListener) listener); + } + } + } + + public void addListener(IInvokedMethodListener listener) + { + m_invokedMethodListeners.add(listener); + } + + public void addListener(ISuiteListener listener) + { + if (null != listener) { + m_suiteListeners.add(listener); + } + } + + public void addListener(ITestListener listener) + { + if (null != listener) { + m_testListeners.add(listener); + } + } + + public void addListener(IReporter listener) + { + if (null != listener) { + m_reporters.add(listener); + } + } + + public void addInvokedMethodListener(IInvokedMethodListener listener) + { + m_invokedMethodListeners.add(listener); + } + + public Set getReporters() + { + return m_reporters; + } + + public List getTestListeners() + { + return m_testListeners; + } + + public List getSuiteListeners() + { + return m_suiteListeners; + } + + /** + * If m_verbose gets set, it will override the verbose setting in testng.xml + */ + private Integer m_verbose = null; + + private final IAnnotationTransformer m_defaultAnnoProcessor = new DefaultAnnotationTransformer(); + private IAnnotationTransformer m_annotationTransformer = m_defaultAnnoProcessor; + + private Boolean m_skipFailedInvocationCounts = false; + + private IMethodInterceptor m_methodInterceptor = null; + + /** + * The list of test names to run from the given suite + */ + private List m_testNames; + + private Integer m_suiteThreadPoolSize = CommandLineArgs.SUITE_THREAD_POOL_SIZE_DEFAULT; + + private boolean m_randomizeSuites = Boolean.FALSE; + + private boolean m_preserveOrder = false; + private Boolean m_groupByInstances; + + private IConfiguration m_configuration; + + /** + * Sets the level of verbosity. This value will override the value specified + * in the test suites. + * + * @param verbose the verbosity level (0 to 10 where 10 is most detailed) + * Actually, this is a lie: you can specify -1 and this will put TestNG + * in debug mode (no longer slicing off stack traces and all). + */ + public void setVerbose(int verbose) + { + m_verbose = verbose; + } + + private void initializeCommandLineSuites() + { + if (m_commandLineTestClasses != null || m_commandLineMethods != null) { + if (null != m_commandLineMethods) { + m_cmdlineSuites = createCommandLineSuitesForMethods(m_commandLineMethods); + } else { + m_cmdlineSuites = createCommandLineSuitesForClasses(m_commandLineTestClasses); + } + + for (XmlSuite s : m_cmdlineSuites) { + for (XmlTest t : s.getTests()) { + t.setPreserveOrder(m_preserveOrder ? "true " : "false"); + } + m_suites.add(s); + if (m_groupByInstances != null) { + s.setGroupByInstances(m_groupByInstances); + } + } + } + } + + private void initializeCommandLineSuitesParams() + { + if (null == m_cmdlineSuites) { + return; + } + + for (XmlSuite s : m_cmdlineSuites) { + if (m_useThreadCount) { + s.setThreadCount(m_threadCount); + } + if (m_useParallelMode) { + s.setParallel(m_parallelMode); + } + if (m_configFailurePolicy != null) { + s.setConfigFailurePolicy(m_configFailurePolicy.toString()); + } + } + + } + + private void initializeCommandLineSuitesGroups() + { + // If groups were specified on the command line, they should override groups + // specified in the XML file + boolean hasIncludedGroups = null != m_includedGroups && m_includedGroups.length > 0; + boolean hasExcludedGroups = null != m_excludedGroups && m_excludedGroups.length > 0; + List suites = m_cmdlineSuites != null ? m_cmdlineSuites : m_suites; + if (hasIncludedGroups || hasExcludedGroups) { + for (XmlSuite s : suites) { + //set on each test, instead of just the first one of the suite + for (XmlTest t : s.getTests()) { + if (hasIncludedGroups) { + t.setIncludedGroups(Arrays.asList(m_includedGroups)); + } + if (hasExcludedGroups) { + t.setExcludedGroups(Arrays.asList(m_excludedGroups)); + } + } + } + } + } + + private void addReporter(Class r) + { + if (!m_reporters.contains(r)) { + m_reporters.add(ClassHelper.newInstance(r)); + } + } + + private void initializeDefaultListeners() + { + m_testListeners.add(new ExitCodeListener(this)); + + if (m_useDefaultListeners) { + addReporter(SuiteHTMLReporter.class); + addReporter(Main.class); + addReporter(FailedReporter.class); + addReporter(XMLReporter.class); + if (System.getProperty("oldTestngEmailableReporter") != null) { + addReporter(EmailableReporter.class); + } else if (System.getProperty("noEmailableReporter") == null) { + addReporter(EmailableReporter2.class); + } + addReporter(JUnitReportReporter.class); + if (m_verbose != null && m_verbose > 4) { + addListener(new VerboseReporter("[TestNG] ")); + } + } + } + + private void initializeConfiguration() + { + ITestObjectFactory factory = m_objectFactory; + // + // Install the listeners found in ServiceLoader (or use the class + // loader for tests, if specified). + // + addServiceLoaderListeners(); + + // + // Install the listeners found in the suites + // + for (XmlSuite s : m_suites) { + for (String listenerName : s.getListeners()) { + Class listenerClass = ClassHelper.forName(listenerName); + + // If specified listener does not exist, a TestNGException will be thrown + if (listenerClass == null) { + throw new TestNGException( + "Listener " + listenerName + + " was not found in project's classpath" + ); + } + + Object listener = ClassHelper.newInstance(listenerClass); + addListener(listener); + } + + // + // Install the method selectors + // + for (XmlMethodSelector methodSelector : s.getMethodSelectors()) { + addMethodSelector(methodSelector.getClassName(), methodSelector.getPriority()); + } + + // + // Find if we have an object factory + // + if (s.getObjectFactory() != null) { + if (factory == null) { + factory = s.getObjectFactory(); + } else { + throw new TestNGException("Found more than one object-factory tag in your suites"); + } + } + } + + m_configuration.setAnnotationFinder(new JDK15AnnotationFinder(getAnnotationTransformer())); + m_configuration.setHookable(m_hookable); + m_configuration.setConfigurable(m_configurable); + m_configuration.setObjectFactory(factory); + } + + /** + * Using reflection to remain Java 5 compliant. + */ + private void addServiceLoaderListeners() + { + try { + Class c = Class.forName("java.util.ServiceLoader"); + List parameters = Lists.newArrayList(); + parameters.add(ITestNGListener.class); + Method loadMethod; + if (m_serviceLoaderClassLoader != null) { + parameters.add(m_serviceLoaderClassLoader); + loadMethod = c.getMethod("load", Class.class, ClassLoader.class); + } else { + loadMethod = c.getMethod("load", Class.class); + } + Iterable loader = + (Iterable) loadMethod.invoke(c, parameters.toArray()); +// Object loader = c. +// ServiceLoader loader = m_serviceLoaderClassLoader != null +// ? ServiceLoader.load(ITestNGListener.class, m_serviceLoaderClassLoader) +// : ServiceLoader.load(ITestNGListener.class); + for (ITestNGListener l : loader) { + Utils.log("[TestNG]", 2, "Adding ServiceLoader listener:" + l); + addListener(l); + addServiceLoaderListener(l); + } + } + catch (ClassNotFoundException ex) { + // Ignore + } + catch (NoSuchMethodException ex) { + // Ignore + } + catch (IllegalAccessException ex) { + // Ignore + } + catch (InvocationTargetException ex) { + // Ignore + } + } + + /** + * Before suites are executed, do a sanity check to ensure all required + * conditions are met. If not, throw an exception to stop test execution + * + * @throws TestNGException if the sanity check fails + */ + private void sanityCheck() + { + checkTestNames(m_suites); + checkSuiteNames(m_suites); + } + + /** + * Ensure that two XmlTest within the same XmlSuite don't have the same name + */ + private void checkTestNames(List suites) + { + for (XmlSuite suite : suites) { + Set testNames = Sets.newHashSet(); + for (XmlTest test : suite.getTests()) { + if (testNames.contains(test.getName())) { + throw new TestNGException( + "Two tests in the same suite " + + "cannot have the same name: " + test.getName() + ); + } else { + testNames.add(test.getName()); + } + } + checkTestNames(suite.getChildSuites()); + } + } + + /** + * Ensure that two XmlSuite don't have the same name + * Otherwise will be clash in SuiteRunnerMap + * See issue #302 + */ + private void checkSuiteNames(List suites) + { + checkSuiteNamesInternal(suites, Sets.newHashSet()); + } + + private void checkSuiteNamesInternal(List suites, Set names) + { + for (XmlSuite suite : suites) { + final String name = suite.getName(); + if (names.contains(name)) { + throw new TestNGException("Two suites cannot have the same name: " + name); + } + names.add(name); + checkSuiteNamesInternal(suite.getChildSuites(), names); + } + } + + /** + * Run TestNG. + */ + public void run() + { + initializeSuitesAndJarFile(); + initializeConfiguration(); + initializeDefaultListeners(); + initializeCommandLineSuites(); + initializeCommandLineSuitesParams(); + initializeCommandLineSuitesGroups(); + + sanityCheck(); + + List suiteRunners = null; + + runExecutionListeners(true /* start */); + + m_start = System.currentTimeMillis(); + + // + // Slave mode + // + if (m_slavefileName != null) { + SuiteSlave slave = new SuiteSlave(m_slavefileName, this); + slave.waitForSuites(); + } + + // + // Regular mode + // + else if (m_masterfileName == null) { + suiteRunners = runSuitesLocally(); + } + + // + // Master mode + // + else { + SuiteDispatcher dispatcher = new SuiteDispatcher(m_masterfileName); + suiteRunners = dispatcher.dispatch( + getConfiguration(), + m_suites, getOutputDirectory(), + getTestListeners() + ); + } + + m_end = System.currentTimeMillis(); + runExecutionListeners(false /* finish */); + + if (null != suiteRunners) { + generateReports(suiteRunners); + } + + if (!m_hasTests) { + setStatus(HAS_NO_TEST); + if (TestRunner.getVerbose() > 1) { + System.err.println("[TestNG] No tests found. Nothing was run"); + usage(); + } + } + } + + private void p(String string) + { + System.out.println("[TestNG] " + string); + } + + private void runExecutionListeners(boolean start) + { + for (List listeners + : Arrays.asList(m_executionListeners, m_configuration.getExecutionListeners())) { + for (IExecutionListener l : listeners) { + if (start) { + l.onExecutionStart(); + } else { + l.onExecutionFinish(); + } + } + } + } + + public void addExecutionListener(IExecutionListener l) + { + m_executionListeners.add(l); + } + + private static void usage() + { + if (m_jCommander == null) { + m_jCommander = new JCommander(new CommandLineArgs()); + } + m_jCommander.usage(); + } + + private void generateReports(List suiteRunners) + { + for (IReporter reporter : m_reporters) { + try { + long start = System.currentTimeMillis(); + reporter.generateReport(m_suites, suiteRunners, m_outputDir); + Utils.log( + "TestNG", 2, "Time taken by " + reporter + ": " + + (System.currentTimeMillis() - start) + " ms" + ); + } + catch (Exception ex) { + System.err.println("[TestNG] Reporter " + reporter + " failed"); + ex.printStackTrace(System.err); + } + } + } + + /** + * This needs to be public for maven2, for now..At least + * until an alternative mechanism is found. + */ + public List runSuitesLocally() + { + SuiteRunnerMap suiteRunnerMap = new SuiteRunnerMap(); + if (m_suites.size() > 0) { + if (m_suites.get(0).getVerbose() >= 2) { + Version.displayBanner(); + } + + // First initialize the suite runners to ensure there are no configuration issues. + // Create a map with XmlSuite as key and corresponding SuiteRunner as value + for (XmlSuite xmlSuite : m_suites) { + createSuiteRunners(suiteRunnerMap, xmlSuite); + } + + // + // Run suites + // + if (m_suiteThreadPoolSize == 1 && !m_randomizeSuites) { + // Single threaded and not randomized: run the suites in order + for (XmlSuite xmlSuite : m_suites) { + runSuitesSequentially( + xmlSuite, suiteRunnerMap, getVerbose(xmlSuite), + getDefaultSuiteName() + ); + } + } else { + // Multithreaded: generate a dynamic graph that stores the suite hierarchy. This is then + // used to run related suites in specific order. Parent suites are run only + // once all the child suites have completed execution + DynamicGraph suiteGraph = new DynamicGraph(); + for (XmlSuite xmlSuite : m_suites) { + populateSuiteGraph(suiteGraph, suiteRunnerMap, xmlSuite); + } + + IThreadWorkerFactory factory = new SuiteWorkerFactory( + suiteRunnerMap, + 0 /* verbose hasn't been set yet */, getDefaultSuiteName() + ); + GraphThreadPoolExecutor pooledExecutor = + new GraphThreadPoolExecutor( + suiteGraph, factory, m_suiteThreadPoolSize, + m_suiteThreadPoolSize, Integer.MAX_VALUE, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue() + ); + + Utils.log("TestNG", 2, "Starting executor for all suites"); + // Run all suites in parallel + pooledExecutor.run(); + try { + pooledExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + pooledExecutor.shutdownNow(); + } + catch (InterruptedException handled) { + Thread.currentThread().interrupt(); + error("Error waiting for concurrent executors to finish " + handled.getMessage()); + } + } + } else { + setStatus(HAS_NO_TEST); + error("No test suite found. Nothing to run"); + usage(); + } + + // + // Generate the suites report + // + return Lists.newArrayList(suiteRunnerMap.values()); + } + + private static void error(String s) + { + LOGGER.error(s); + } + + /** + * @return the verbose level, checking in order: the verbose level on + * the suite, the verbose level on the TestNG object, or 1. + */ + private int getVerbose(XmlSuite xmlSuite) + { + int result = xmlSuite.getVerbose() != null ? xmlSuite.getVerbose() + : (m_verbose != null ? m_verbose : DEFAULT_VERBOSE); + return result; + } + + /** + * Recursively runs suites. Runs the children suites before running the parent + * suite. This is done so that the results for parent suite can reflect the + * combined results of the children suites. + * + * @param xmlSuite XML Suite to be executed + * @param suiteRunnerMap Maps {@code XmlSuite}s to respective {@code ISuite} + * @param verbose verbose level + * @param defaultSuiteName default suite name + */ + private void runSuitesSequentially( + XmlSuite xmlSuite, + SuiteRunnerMap suiteRunnerMap, int verbose, String defaultSuiteName + ) + { + for (XmlSuite childSuite : xmlSuite.getChildSuites()) { + runSuitesSequentially(childSuite, suiteRunnerMap, verbose, defaultSuiteName); + } + SuiteRunnerWorker srw = new SuiteRunnerWorker( + suiteRunnerMap.get(xmlSuite), suiteRunnerMap, + verbose, defaultSuiteName + ); + srw.run(); + } + + /** + * Populates the dynamic graph with the reverse hierarchy of suites. Edges are + * added pointing from child suite runners to parent suite runners, hence making + * parent suite runners dependent on all the child suite runners + * + * @param suiteGraph dynamic graph representing the reverse hierarchy of SuiteRunners + * @param suiteRunnerMap Map with XMLSuite as key and its respective SuiteRunner as value + * @param xmlSuite XML Suite + */ + private void populateSuiteGraph( + DynamicGraph suiteGraph /* OUT */, + SuiteRunnerMap suiteRunnerMap, XmlSuite xmlSuite + ) + { + ISuite parentSuiteRunner = suiteRunnerMap.get(xmlSuite); + if (xmlSuite.getChildSuites().isEmpty()) { + suiteGraph.addNode(parentSuiteRunner); + } else { + for (XmlSuite childSuite : xmlSuite.getChildSuites()) { + suiteGraph.addEdge(parentSuiteRunner, suiteRunnerMap.get(childSuite)); + populateSuiteGraph(suiteGraph, suiteRunnerMap, childSuite); + } + } + } + + /** + * Creates the {@code SuiteRunner}s and populates the suite runner map with + * this information + * + * @param suiteRunnerMap Map with XMLSuite as key and it's respective + * SuiteRunner as value. This is updated as part of this method call + * @param xmlSuite Xml Suite (and its children) for which {@code SuiteRunner}s are created + */ + private void createSuiteRunners(SuiteRunnerMap suiteRunnerMap /* OUT */, XmlSuite xmlSuite) + { + if (null != m_isJUnit && !m_isJUnit.equals(XmlSuite.DEFAULT_JUNIT)) { + xmlSuite.setJUnit(m_isJUnit); + } + + // If the skip flag was invoked on the command line, it + // takes precedence + if (null != m_skipFailedInvocationCounts) { + xmlSuite.setSkipFailedInvocationCounts(m_skipFailedInvocationCounts); + } + + // Override the XmlSuite verbose value with the one from TestNG + if (m_verbose != null) { + xmlSuite.setVerbose(m_verbose); + } + + if (null != m_configFailurePolicy) { + xmlSuite.setConfigFailurePolicy(m_configFailurePolicy); + } + + for (XmlTest t : xmlSuite.getTests()) { + for (Map.Entry ms : m_methodDescriptors.entrySet()) { + XmlMethodSelector xms = new XmlMethodSelector(); + xms.setName(ms.getKey()); + xms.setPriority(ms.getValue()); + t.getMethodSelectors().add(xms); + } + } + + suiteRunnerMap.put(xmlSuite, createSuiteRunner(xmlSuite)); + + for (XmlSuite childSuite : xmlSuite.getChildSuites()) { + createSuiteRunners(suiteRunnerMap, childSuite); + } + } + + /** + * Creates a suite runner and configures its initial state + * + * @param xmlSuite + * + * @return returns the newly created suite runner + */ + private SuiteRunner createSuiteRunner(XmlSuite xmlSuite) + { + SuiteRunner result = new SuiteRunner( + getConfiguration(), xmlSuite, + m_outputDir, + m_testRunnerFactory, + m_useDefaultListeners, + m_methodInterceptor, + m_invokedMethodListeners, + m_testListeners + ); + + for (ISuiteListener isl : m_suiteListeners) { + result.addListener(isl); + } + + for (IReporter r : result.getReporters()) { + addListener(r); + } + + for (IConfigurationListener cl : m_configuration.getConfigurationListeners()) { + result.addListener(cl); + } + + return result; + } + + protected IConfiguration getConfiguration() + { + return m_configuration; + } + + /** + * The TestNG entry point for command line execution. + * + * @param argv the TestNG command line parameters. + * + * @throws FileNotFoundException + */ + public static void main(String[] argv) + { + TestNG testng = privateMain(argv, null); + System.exit(testng.getStatus()); + } + + /** + * Note: this method is not part of the public API and is meant for internal usage only. + */ + public static TestNG privateMain(String[] argv, ITestListener listener) + { + TestNG result = new TestNG(); + + if (null != listener) { + result.addListener(listener); + } + + // + // Parse the arguments + // + try { + CommandLineArgs cla = new CommandLineArgs(); + m_jCommander = new JCommander(cla, argv); + validateCommandLineParameters(cla); + result.configure(cla); + } + catch (ParameterException ex) { + exitWithError(ex.getMessage()); + } + + // + // Run + // + try { + result.run(); + } + catch (TestNGException ex) { + if (TestRunner.getVerbose() > 1) { + ex.printStackTrace(System.out); + } else { + error(ex.getMessage()); + } + result.setStatus(HAS_FAILURE); + } + + return result; + } + + /** + * Configure the TestNG instance based on the command line parameters. + */ + protected void configure(CommandLineArgs cla) + { + if (cla.verbose != null) { + setVerbose(cla.verbose); + } + setOutputDirectory(cla.outputDirectory); + + String testClasses = cla.testClass; + if (null != testClasses) { + String[] strClasses = testClasses.split(","); + List classes = Lists.newArrayList(); + for (String c : strClasses) { + classes.add(ClassHelper.fileToClass(c)); + } + + setTestClasses(classes.toArray(new Class[classes.size()])); + } + + setOutputDirectory(cla.outputDirectory); + + if (cla.testNames != null) { + setTestNames(Arrays.asList(cla.testNames.split(","))); + } + +// List testNgXml = (List) cmdLineArgs.get(CommandLineArgs.SUITE_DEF); +// if (null != testNgXml) { +// setTestSuites(testNgXml); +// } + + // Note: can't use a Boolean field here because we are allowing a boolean + // parameter with an arity of 1 ("-usedefaultlisteners false") + if (cla.useDefaultListeners != null) { + setUseDefaultListeners("true".equalsIgnoreCase(cla.useDefaultListeners)); + } + + setGroups(cla.groups); + setExcludedGroups(cla.excludedGroups); + setTestJar(cla.testJar); + setXmlPathInJar(cla.xmlPathInJar); + setJUnit(cla.junit); + setMixed(cla.mixed); + setMaster(cla.master); + setSlave(cla.slave); + setSkipFailedInvocationCounts(cla.skipFailedInvocationCounts); + if (cla.parallelMode != null) { + setParallel(cla.parallelMode); + } + if (cla.configFailurePolicy != null) { + setConfigFailurePolicy(cla.configFailurePolicy); + } + if (cla.threadCount != null) { + setThreadCount(cla.threadCount); + } + if (cla.dataProviderThreadCount != null) { + setDataProviderThreadCount(cla.dataProviderThreadCount); + } + if (cla.suiteName != null) { + setDefaultSuiteName(cla.suiteName); + } + if (cla.testName != null) { + setDefaultTestName(cla.testName); + } + if (cla.listener != null) { + String sep = ";"; + if (cla.listener.indexOf(",") >= 0) { + sep = ","; + } + String[] strs = Utils.split(cla.listener, sep); + List classes = Lists.newArrayList(); + + for (String cls : strs) { + classes.add(ClassHelper.fileToClass(cls)); + } + + setListenerClasses(classes); + } + + if (null != cla.methodSelectors) { + String[] strs = Utils.split(cla.methodSelectors, ","); + for (String cls : strs) { + String[] sel = Utils.split(cls, ":"); + try { + if (sel.length == 2) { + addMethodSelector(sel[0], Integer.valueOf(sel[1])); + } else { + error("Method selector value was not in the format org.example.Selector:4"); + } + } + catch (NumberFormatException nfe) { + error("Method selector value was not in the format org.example.Selector:4"); + } + } + } + + if (cla.objectFactory != null) { + setObjectFactory(ClassHelper.fileToClass(cla.objectFactory)); + } + if (cla.testRunnerFactory != null) { + setTestRunnerFactoryClass( + ClassHelper.fileToClass(cla.testRunnerFactory) + ); + } + //######### PATCH Begins + if (System.getProperty("testrunfactory") != null) { + setTestRunnerFactoryClass( + ClassHelper.fileToClass(System.getProperty("testrunfactory")) + ); + } + + //######### PATCH ends + + if (cla.reporter != null) { + ReporterConfig reporterConfig = ReporterConfig.deserialize(cla.reporter); + addReporter(reporterConfig); + } + + if (cla.commandLineMethods.size() > 0) { + m_commandLineMethods = cla.commandLineMethods; + } + + if (cla.suiteFiles != null) { + setTestSuites(cla.suiteFiles); + } + + setSuiteThreadPoolSize(cla.suiteThreadPoolSize); + setRandomizeSuites(cla.randomizeSuites); + } + + public void setSuiteThreadPoolSize(Integer suiteThreadPoolSize) + { + m_suiteThreadPoolSize = suiteThreadPoolSize; + } + + public Integer getSuiteThreadPoolSize() + { + return m_suiteThreadPoolSize; + } + + public void setRandomizeSuites(boolean randomizeSuites) + { + m_randomizeSuites = randomizeSuites; + } + + /** + * This method is invoked by Maven's Surefire, only remove it once + * Surefire has been modified to no longer call it. + */ + public void setSourcePath(String path) + { + // nop + } + + /** + * This method is invoked by Maven's Surefire to configure the runner, + * do not remove unless you know for sure that Surefire has been updated + * to use the new configure(CommandLineArgs) method. + * + * @deprecated use new configure(CommandLineArgs) method + */ + @SuppressWarnings({"unchecked"}) + @Deprecated + public void configure(Map cmdLineArgs) + { + CommandLineArgs result = new CommandLineArgs(); + + Integer verbose = (Integer) cmdLineArgs.get(CommandLineArgs.LOG); + if (null != verbose) { + result.verbose = verbose; + } + result.outputDirectory = (String) cmdLineArgs.get(CommandLineArgs.OUTPUT_DIRECTORY); + + String testClasses = (String) cmdLineArgs.get(CommandLineArgs.TEST_CLASS); + if (null != testClasses) { + result.testClass = testClasses; + } + + String testNames = (String) cmdLineArgs.get(CommandLineArgs.TEST_NAMES); + if (testNames != null) { + result.testNames = testNames; + } + + String useDefaultListeners = (String) cmdLineArgs.get(CommandLineArgs.USE_DEFAULT_LISTENERS); + if (null != useDefaultListeners) { + result.useDefaultListeners = useDefaultListeners; + } + + result.groups = (String) cmdLineArgs.get(CommandLineArgs.GROUPS); + result.excludedGroups = (String) cmdLineArgs.get(CommandLineArgs.EXCLUDED_GROUPS); + result.testJar = (String) cmdLineArgs.get(CommandLineArgs.TEST_JAR); + result.xmlPathInJar = (String) cmdLineArgs.get(CommandLineArgs.XML_PATH_IN_JAR); + result.junit = (Boolean) cmdLineArgs.get(CommandLineArgs.JUNIT); + result.mixed = (Boolean) cmdLineArgs.get(CommandLineArgs.MIXED); + result.master = (String) cmdLineArgs.get(CommandLineArgs.MASTER); + result.slave = (String) cmdLineArgs.get(CommandLineArgs.SLAVE); + result.skipFailedInvocationCounts = (Boolean) cmdLineArgs.get( + CommandLineArgs.SKIP_FAILED_INVOCATION_COUNTS + ); + String parallelMode = (String) cmdLineArgs.get(CommandLineArgs.PARALLEL); + if (parallelMode != null) { + result.parallelMode = parallelMode; + } + + String threadCount = (String) cmdLineArgs.get(CommandLineArgs.THREAD_COUNT); + if (threadCount != null) { + result.threadCount = Integer.parseInt(threadCount); + } + + // Not supported by Surefire yet + Integer dptc = (Integer) cmdLineArgs.get(CommandLineArgs.DATA_PROVIDER_THREAD_COUNT); + if (dptc != null) { + result.dataProviderThreadCount = dptc; + } + String defaultSuiteName = (String) cmdLineArgs.get(CommandLineArgs.SUITE_NAME); + if (defaultSuiteName != null) { + result.suiteName = defaultSuiteName; + } + + String defaultTestName = (String) cmdLineArgs.get(CommandLineArgs.TEST_NAME); + if (defaultTestName != null) { + result.testName = defaultTestName; + } + + Object listeners = cmdLineArgs.get(CommandLineArgs.LISTENER); + if (listeners instanceof List) { + result.listener = Utils.join((List) listeners, ","); + } else { + result.listener = (String) listeners; + } + + String ms = (String) cmdLineArgs.get(CommandLineArgs.METHOD_SELECTORS); + if (null != ms) { + result.methodSelectors = ms; + } + + String objectFactory = (String) cmdLineArgs.get(CommandLineArgs.OBJECT_FACTORY); + if (null != objectFactory) { + result.objectFactory = objectFactory; + } + + String runnerFactory = (String) cmdLineArgs.get(CommandLineArgs.TEST_RUNNER_FACTORY); + if (null != runnerFactory) { + result.testRunnerFactory = runnerFactory; + } + + String reporterConfigs = (String) cmdLineArgs.get(CommandLineArgs.REPORTER); + if (reporterConfigs != null) { + result.reporter = reporterConfigs; + } + + String failurePolicy = (String) cmdLineArgs.get(CommandLineArgs.CONFIG_FAILURE_POLICY); + if (failurePolicy != null) { + result.configFailurePolicy = failurePolicy; + } + + configure(result); + } + + /** + * Only run the specified tests from the suite. + */ + public void setTestNames(List testNames) + { + m_testNames = testNames; + } + + public void setSkipFailedInvocationCounts(Boolean skip) + { + m_skipFailedInvocationCounts = skip; + } + + private void addReporter(ReporterConfig reporterConfig) + { + Object instance = reporterConfig.newReporterInstance(); + if (instance != null) { + addListener(instance); + } else { + LOGGER.warn("Could not find reporte class : " + reporterConfig.getClassName()); + } + } + + /** + * Specify if this run should be in Master-Slave mode as Master + * + * @param fileName remote.properties path + */ + public void setMaster(String fileName) + { + m_masterfileName = fileName; + } + + /** + * Specify if this run should be in Master-Slave mode as slave + * + * @param fileName remote.properties path + */ + public void setSlave(String fileName) + { + m_slavefileName = fileName; + } + + /** + * Specify if this run should be made in JUnit mode + * + * @param isJUnit + */ + public void setJUnit(Boolean isJUnit) + { + m_isJUnit = isJUnit; + } + + /** + * Specify if this run should be made in mixed mode + */ + public void setMixed(Boolean isMixed) + { + if (isMixed == null) { + return; + } + m_isMixed = isMixed; + } + + /** + * @deprecated The TestNG version is now established at load time. This + * method is not required anymore and is now a no-op. + */ + @Deprecated + public static void setTestNGVersion() + { + LOGGER.info("setTestNGVersion has been deprecated."); + } + + /** + * Returns true if this is the JDK 1.4 JAR version of TestNG, false otherwise. + * + * @return true if this is the JDK 1.4 JAR version of TestNG, false otherwise. + */ + @Deprecated + public static boolean isJdk14() + { + return false; + } + + /** + * Double check that the command line parameters are valid. + */ + protected static void validateCommandLineParameters(CommandLineArgs args) + { + String testClasses = args.testClass; + List testNgXml = args.suiteFiles; + String testJar = args.testJar; + String slave = args.slave; + List methods = args.commandLineMethods; + + if (testClasses == null && slave == null && testJar == null + && (testNgXml == null || testNgXml.isEmpty()) + && (methods == null || methods.isEmpty())) { + throw new ParameterException( + "You need to specify at least one testng.xml, one class" + + " or one method" + ); + } + + String groups = args.groups; + String excludedGroups = args.excludedGroups; + + if (testJar == null && + (null != groups || null != excludedGroups) && testClasses == null + && (testNgXml == null || testNgXml.isEmpty())) { + throw new ParameterException("Groups option should be used with testclass option"); + } + + if (args.slave != null && args.master != null) { + throw new ParameterException( + CommandLineArgs.SLAVE + " can't be combined with " + + CommandLineArgs.MASTER + ); + } + + Boolean junit = args.junit; + Boolean mixed = args.mixed; + if (junit && mixed) { + throw new ParameterException( + CommandLineArgs.MIXED + " can't be combined with " + + CommandLineArgs.JUNIT + ); + } + } + + /** + * @return true if at least one test failed. + */ + public boolean hasFailure() + { + return (getStatus() & HAS_FAILURE) == HAS_FAILURE; + } + + /** + * @return true if at least one test failed within success percentage. + */ + public boolean hasFailureWithinSuccessPercentage() + { + return (getStatus() & HAS_FSP) == HAS_FSP; + } + + /** + * @return true if at least one test was skipped. + */ + public boolean hasSkip() + { + return (getStatus() & HAS_SKIPPED) == HAS_SKIPPED; + } + + static void exitWithError(String msg) + { + System.err.println(msg); + usage(); + System.exit(1); + } + + public String getOutputDirectory() + { + return m_outputDir; + } + + public IAnnotationTransformer getAnnotationTransformer() + { + return m_annotationTransformer; + } + + public void setAnnotationTransformer(IAnnotationTransformer t) + { + // compare by reference! + if (m_annotationTransformer != m_defaultAnnoProcessor && m_annotationTransformer != t) { + LOGGER.warn("AnnotationTransformer already set"); + } + m_annotationTransformer = t; + } + + /** + * @return the defaultSuiteName + */ + public String getDefaultSuiteName() + { + return m_defaultSuiteName; + } + + /** + * @param defaultSuiteName the defaultSuiteName to set + */ + public void setDefaultSuiteName(String defaultSuiteName) + { + m_defaultSuiteName = defaultSuiteName; + } + + /** + * @return the defaultTestName + */ + public String getDefaultTestName() + { + return m_defaultTestName; + } + + /** + * @param defaultTestName the defaultTestName to set + */ + public void setDefaultTestName(String defaultTestName) + { + m_defaultTestName = defaultTestName; + } + + /** + * Sets the policy for whether or not to ever invoke a configuration method again after + * it has failed once. Possible values are defined in {@link XmlSuite}. The default + * value is {@link XmlSuite#SKIP}. + * + * @param failurePolicy the configuration failure policy + */ + public void setConfigFailurePolicy(String failurePolicy) + { + m_configFailurePolicy = failurePolicy; + } + + /** + * Returns the configuration failure policy. + * + * @return config failure policy + */ + public String getConfigFailurePolicy() + { + return m_configFailurePolicy; + } + + // DEPRECATED: to be removed after a major version change + + /** + * @deprecated since 5.1 + */ + @Deprecated + public static TestNG getDefault() + { + return m_instance; + } + + /** + * @deprecated since 5.1 + */ + @Deprecated + public void setHasFailure(boolean hasFailure) + { + m_status |= HAS_FAILURE; + } + + /** + * @deprecated since 5.1 + */ + @Deprecated + public void setHasFailureWithinSuccessPercentage(boolean hasFailureWithinSuccessPercentage) + { + m_status |= HAS_FSP; + } + + /** + * @deprecated since 5.1 + */ + @Deprecated + public void setHasSkip(boolean hasSkip) + { + m_status |= HAS_SKIPPED; + } + + public static class ExitCodeListener implements IResultListener2 + { + private TestNG m_mainRunner; + + public ExitCodeListener() + { + m_mainRunner = TestNG.m_instance; + } + + public ExitCodeListener(TestNG runner) + { + m_mainRunner = runner; + } + + @Override + public void beforeConfiguration(ITestResult tr) + { + } + + @Override + public void onTestFailure(ITestResult result) + { + setHasRunTests(); + m_mainRunner.setStatus(HAS_FAILURE); + } + + @Override + public void onTestSkipped(ITestResult result) + { + setHasRunTests(); + m_mainRunner.setStatus(HAS_SKIPPED); + } + + @Override + public void onTestFailedButWithinSuccessPercentage(ITestResult result) + { + setHasRunTests(); + m_mainRunner.setStatus(HAS_FSP); + } + + @Override + public void onTestSuccess(ITestResult result) + { + setHasRunTests(); + } + + @Override + public void onStart(ITestContext context) + { + setHasRunTests(); + } + + @Override + public void onFinish(ITestContext context) + { + } + + @Override + public void onTestStart(ITestResult result) + { + setHasRunTests(); + } + + private void setHasRunTests() + { + m_mainRunner.m_hasTests = true; + } + + /** + * @see org.testng.IConfigurationListener#onConfigurationFailure(org.testng.ITestResult) + */ + @Override + public void onConfigurationFailure(ITestResult itr) + { + m_mainRunner.setStatus(HAS_FAILURE); + } + + /** + * @see org.testng.IConfigurationListener#onConfigurationSkip(org.testng.ITestResult) + */ + @Override + public void onConfigurationSkip(ITestResult itr) + { + m_mainRunner.setStatus(HAS_SKIPPED); + } + + /** + * @see org.testng.IConfigurationListener#onConfigurationSuccess(org.testng.ITestResult) + */ + @Override + public void onConfigurationSuccess(ITestResult itr) + { + } + } + + private void setConfigurable(IConfigurable c) + { + // compare by reference! + if (m_configurable != null && m_configurable != c) { + LOGGER.warn("Configurable already set"); + } + m_configurable = c; + } + + private void setHookable(IHookable h) + { + // compare by reference! + if (m_hookable != null && m_hookable != h) { + LOGGER.warn("Hookable already set"); + } + m_hookable = h; + } + + public void setMethodInterceptor(IMethodInterceptor i) + { + // compare by reference! + if (m_methodInterceptor != null && m_methodInterceptor != i) { + LOGGER.warn("MethodInterceptor already set"); + } + m_methodInterceptor = i; + } + + public void setDataProviderThreadCount(int count) + { + m_dataProviderThreadCount = count; + } + + /** + * Add a class loader to the searchable loaders. + */ + public void addClassLoader(final ClassLoader loader) + { + if (loader != null) { + ClassHelper.addClassLoader(loader); + } + } + + public void setPreserveOrder(boolean b) + { + m_preserveOrder = b; + } + + protected long getStart() + { + return m_start; + } + + protected long getEnd() + { + return m_end; + } + + public void setGroupByInstances(boolean b) + { + m_groupByInstances = b; + } + + ///// + // ServiceLoader testing + // + + private URLClassLoader m_serviceLoaderClassLoader; + private List m_serviceLoaderListeners = Lists.newArrayList(); + + /* + * Used to test ServiceClassLoader + */ + public void setServiceLoaderClassLoader(URLClassLoader ucl) + { + m_serviceLoaderClassLoader = ucl; + } + + /* + * Used to test ServiceClassLoader + */ + private void addServiceLoaderListener(ITestNGListener l) + { + m_serviceLoaderListeners.add(l); + } + + /* + * Used to test ServiceClassLoader + */ + public List getServiceLoaderListeners() + { + return m_serviceLoaderListeners; + } + + // + // ServiceLoader testing + ///// +} diff --git a/integration-tests/src/main/java/org/testng/remote/RemoteTestNG.java b/integration-tests/src/main/java/org/testng/remote/RemoteTestNG.java new file mode 100644 index 00000000000..8c7fc8d6db2 --- /dev/null +++ b/integration-tests/src/main/java/org/testng/remote/RemoteTestNG.java @@ -0,0 +1,336 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.testng.remote; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.ParameterException; +import org.testng.CommandLineArgs; +import org.testng.IInvokedMethodListener; +import org.testng.ISuite; +import org.testng.ISuiteListener; +import org.testng.ITestRunnerFactory; +import org.testng.TestNG; +import org.testng.TestNGException; +import org.testng.TestRunner; +import org.testng.collections.Lists; +import org.testng.internal.ClassHelper; +import org.testng.remote.strprotocol.GenericMessage; +import org.testng.remote.strprotocol.IMessageSender; +import org.testng.remote.strprotocol.MessageHelper; +import org.testng.remote.strprotocol.MessageHub; +import org.testng.remote.strprotocol.RemoteTestListener; +import org.testng.remote.strprotocol.SerializedMessageSender; +import org.testng.remote.strprotocol.StringMessageSender; +import org.testng.remote.strprotocol.SuiteMessage; +import org.testng.reporters.JUnitXMLReporter; +import org.testng.reporters.TestHTMLReporter; +import org.testng.xml.XmlSuite; +import org.testng.xml.XmlTest; + +import java.util.Arrays; +import java.util.List; + +import static org.testng.internal.Utils.defaultIfStringEmpty; + +/** + * Class copied from TestNG library ver 6.8.7 to apply a workaround for http://jira.codehaus.org/browse/SUREFIRE-622 + * To Locate the PATCHED AREA search for keyword "PATCH" in this class file + *

+ * Extension of TestNG registering a remote TestListener. + * + * @author Cedric Beust + */ +public class RemoteTestNG extends TestNG +{ + // The following constants are referenced by the Eclipse plug-in, make sure you + // modify the plug-in as well if you change any of them. + public static final String DEBUG_PORT = "12345"; + public static final String DEBUG_SUITE_FILE = "testng-customsuite.xml"; + public static final String DEBUG_SUITE_DIRECTORY = System.getProperty("java.io.tmpdir"); + public static final String PROPERTY_DEBUG = "testng.eclipse.debug"; + public static final String PROPERTY_VERBOSE = "testng.eclipse.verbose"; + private static final String LOCALHOST = "localhost"; + // End of Eclipse constants. + /** + * Port used for the serialized protocol + */ + private static Integer m_serPort = null; + private static boolean m_debug; + private static boolean m_dontExit; + private static boolean m_ack; + private ITestRunnerFactory m_customTestRunnerFactory; + private String m_host; + /** + * Port used for the string protocol + */ + private Integer m_port = null; + + public static void main(String[] args) throws ParameterException + { + CommandLineArgs cla = new CommandLineArgs(); + RemoteArgs ra = new RemoteArgs(); + new JCommander(Arrays.asList(cla, ra), args); + m_dontExit = ra.dontExit; + if (cla.port != null && ra.serPort != null) { + throw new TestNGException( + "Can only specify one of " + CommandLineArgs.PORT + + " and " + RemoteArgs.PORT + ); + } + m_debug = cla.debug; + m_ack = ra.ack; + if (m_debug) { +// while (true) { + initAndRun(args, cla, ra); +// } + } else { + initAndRun(args, cla, ra); + } + } + + private static void initAndRun(String[] args, CommandLineArgs cla, RemoteArgs ra) + { + RemoteTestNG remoteTestNg = new RemoteTestNG(); + if (m_debug) { + // In debug mode, override the port and the XML file to a fixed location + cla.port = Integer.parseInt(DEBUG_PORT); + ra.serPort = cla.port; + cla.suiteFiles = Arrays.asList( + new String[]{ + DEBUG_SUITE_DIRECTORY + DEBUG_SUITE_FILE + } + ); + } + remoteTestNg.configure(cla); + remoteTestNg.setHost(cla.host); + m_serPort = ra.serPort; + remoteTestNg.m_port = cla.port; + if (isVerbose()) { + StringBuilder sb = new StringBuilder("Invoked with "); + for (String s : args) { + sb.append(s).append(" "); + } + p(sb.toString()); +// remoteTestNg.setVerbose(1); +// } else { +// remoteTestNg.setVerbose(0); + } + validateCommandLineParameters(cla); + remoteTestNg.run(); +// if (m_debug) { +// // Run in a loop if in debug mode so it is possible to run several launches +// // without having to relauch RemoteTestNG. +// while (true) { +// remoteTestNg.run(); +// remoteTestNg.configure(cla); +// } +// } else { +// remoteTestNg.run(); +// } + } + + private static void p(String s) + { + if (isVerbose()) { + System.out.println("[RemoteTestNG] " + s); + } + } + + public static boolean isVerbose() + { + boolean result = System.getProperty(PROPERTY_VERBOSE) != null || isDebug(); + return result; + } + + public static boolean isDebug() + { + return m_debug || System.getProperty(PROPERTY_DEBUG) != null; + } + + private void calculateAllSuites(List suites, List outSuites) + { + for (XmlSuite s : suites) { + outSuites.add(s); +// calculateAllSuites(s.getChildSuites(), outSuites); + } + } + + @Override + public void run() + { + IMessageSender sender = m_serPort != null + ? new SerializedMessageSender(m_host, m_serPort, m_ack) + : new StringMessageSender(m_host, m_port); + final MessageHub msh = new MessageHub(sender); + msh.setDebug(isDebug()); + try { + msh.connect(); + // We couldn't do this until now in debug mode since the .xml file didn't exist yet. + // Now that we have connected with the Eclipse client, we know that it created the .xml + // file so we can proceed with the initialization + initializeSuitesAndJarFile(); + + List suites = Lists.newArrayList(); + calculateAllSuites(m_suites, suites); +// System.out.println("Suites: " + m_suites.get(0).getChildSuites().size() +// + " and:" + suites.get(0).getChildSuites().size()); + if (suites.size() > 0) { + + int testCount = 0; + + for (int i = 0; i < suites.size(); i++) { + testCount += (suites.get(i)).getTests().size(); + } + + GenericMessage gm = new GenericMessage(MessageHelper.GENERIC_SUITE_COUNT); + gm.setSuiteCount(suites.size()); + gm.setTestCount(testCount); + msh.sendMessage(gm); + + addListener(new RemoteSuiteListener(msh)); + setTestRunnerFactory(new DelegatingTestRunnerFactory(buildTestRunnerFactory(), msh)); + +// System.out.println("RemoteTestNG starting"); + super.run(); + } else { + System.err.println("No test suite found. Nothing to run"); + } + } + catch (Throwable cause) { + cause.printStackTrace(System.err); + } + finally { +// System.out.println("RemoteTestNG finishing: " + (getEnd() - getStart()) + " ms"); + msh.shutDown(); + if (!m_debug && !m_dontExit) { + System.exit(0); + } + } + } + + /** + * Override by the plugin if you need to configure differently the TestRunner + * (usually this is needed if different listeners/reporters are needed). + * Note: you don't need to worry about the wiring listener, because it is added + * automatically. + */ + protected ITestRunnerFactory buildTestRunnerFactory() + { + //################### PATCH STARTS + if (System.getProperty("testrunfactory") != null) { + m_customTestRunnerFactory = (ITestRunnerFactory) ClassHelper.newInstance( + ClassHelper.fileToClass( + System.getProperty( + "testrunfactory" + ) + ) + ); + //################## PATCH ENDS + } else if (null == m_customTestRunnerFactory) { + m_customTestRunnerFactory = new ITestRunnerFactory() + { + @Override + public TestRunner newTestRunner( + ISuite suite, XmlTest xmlTest, + List listeners + ) + { + TestRunner runner = + new TestRunner( + getConfiguration(), suite, xmlTest, + false /*skipFailedInvocationCounts */, + listeners + ); + if (m_useDefaultListeners) { + runner.addListener(new TestHTMLReporter()); + runner.addListener(new JUnitXMLReporter()); + } + + return runner; + } + }; + } + + return m_customTestRunnerFactory; + } + + private String getHost() + { + return m_host; + } + + public void setHost(String host) + { + m_host = defaultIfStringEmpty(host, LOCALHOST); + } + + private int getPort() + { + return m_port; + } + + /** + * A ISuiteListener wiring the results using the internal string-based protocol. + */ + private static class RemoteSuiteListener implements ISuiteListener + { + private final MessageHub m_messageSender; + + RemoteSuiteListener(MessageHub smsh) + { + m_messageSender = smsh; + } + + @Override + public void onFinish(ISuite suite) + { + m_messageSender.sendMessage(new SuiteMessage(suite, false /*start*/)); + } + + @Override + public void onStart(ISuite suite) + { + m_messageSender.sendMessage(new SuiteMessage(suite, true /*start*/)); + } + } + + private static class DelegatingTestRunnerFactory implements ITestRunnerFactory + { + private final ITestRunnerFactory m_delegateFactory; + private final MessageHub m_messageSender; + + DelegatingTestRunnerFactory(ITestRunnerFactory trf, MessageHub smsh) + { + m_delegateFactory = trf; + m_messageSender = smsh; + } + + @Override + public TestRunner newTestRunner( + ISuite suite, XmlTest test, + List listeners + ) + { + TestRunner tr = m_delegateFactory.newTestRunner(suite, test, listeners); + tr.addListener(new RemoteTestListener(suite, test, m_messageSender)); + return tr; + } + } +} diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java new file mode 100644 index 00000000000..1e333b8b8d7 --- /dev/null +++ b/integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java @@ -0,0 +1,85 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.tests.indexer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import io.druid.testing.clients.CoordinatorResourceTestClient; +import io.druid.testing.clients.OverlordResourceTestClient; +import io.druid.testing.utils.FromFileTestQueryHelper; +import io.druid.testing.utils.RetryUtil; +import org.apache.commons.io.IOUtils; +import org.joda.time.Interval; + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.util.concurrent.Callable; + +public abstract class AbstractIndexerTest +{ + + @Inject + protected CoordinatorResourceTestClient coordinator; + @Inject + protected OverlordResourceTestClient indexer; + @Inject + protected ObjectMapper jsonMapper; + + @Inject + protected FromFileTestQueryHelper queryHelper; + + protected void unloadAndKillData(final String dataSource) throws Exception + { + Interval interval = new Interval("2013-01-01T00:00:00.000Z/2013-12-01T00:00:00.000Z"); + coordinator.unloadSegmentsForDataSource(dataSource, interval); + RetryUtil.retryUntilFalse( + new Callable() + { + @Override + public Boolean call() throws Exception + { + return coordinator.areSegmentsLoaded(dataSource); + } + }, "Segment Unloading" + ); + coordinator.deleteSegmentsDataSource(dataSource, interval); + RetryUtil.retryUntilTrue( + new Callable() + { + @Override + public Boolean call() throws Exception + { + return (indexer.getPendingTasks().size() + indexer.getRunningTasks().size() + indexer.getWaitingTasks() + .size()) == 0; + } + }, "Waiting for Tasks Completion" + ); + } + + protected String getTaskAsString(String file) throws IOException + { + InputStream inputStream = ITRealtimeIndexTaskTest.class.getResourceAsStream(file); + StringWriter writer = new StringWriter(); + IOUtils.copy(inputStream, writer, "UTF-8"); + return writer.toString(); + } + +} diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java new file mode 100644 index 00000000000..89c68d7ef61 --- /dev/null +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java @@ -0,0 +1,81 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.tests.indexer; + +import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.metamx.common.logger.Logger; +import io.druid.testing.IntegrationTestingConfig; +import io.druid.testing.guice.DruidTestModuleFactory; +import io.druid.testing.utils.RetryUtil; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.util.concurrent.Callable; + +@Guice(moduleFactory = DruidTestModuleFactory.class) +public class ITIndexerTest extends AbstractIndexerTest +{ + private static final Logger LOG = new Logger(ITIndexerTest.class); + private static String INDEX_TASK = "/indexer/wikipedia_index_task.json"; + private static String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_index_queries.json"; + private static String INDEX_DATASOURCE = "wikipedia_index_test"; + + + @Inject + private IntegrationTestingConfig config; + + @Test + public void testIndexData() throws Exception + { + loadData(); + try { + queryHelper.testQueriesFromFile(INDEX_QUERIES_RESOURCE, 2); + } + catch (Exception e) { + e.printStackTrace(); + Throwables.propagate(e); + } + finally { + unloadAndKillData(INDEX_DATASOURCE); + } + + } + + private void loadData() throws Exception + { + final String taskID = indexer.submitTask(getTaskAsString(INDEX_TASK)); + LOG.info("TaskID for loading index task %s", taskID); + indexer.waitUntilTaskCompletes(taskID); + + RetryUtil.retryUntilTrue( + new Callable() + { + @Override + public Boolean call() throws Exception + { + return coordinator.areSegmentsLoaded(INDEX_DATASOURCE); + } + }, "Segment Load" + ); + } + + +} diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java new file mode 100644 index 00000000000..7066a75beae --- /dev/null +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java @@ -0,0 +1,142 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.tests.indexer; + +import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.metamx.common.logger.Logger; +import com.metamx.http.client.HttpClient; +import io.druid.curator.discovery.ServerDiscoveryFactory; +import io.druid.curator.discovery.ServerDiscoverySelector; +import io.druid.guice.annotations.Global; +import io.druid.testing.IntegrationTestingConfig; +import io.druid.testing.clients.EventReceiverFirehoseTestClient; +import io.druid.testing.guice.DruidTestModuleFactory; +import io.druid.testing.utils.RetryUtil; +import io.druid.testing.utils.ServerDiscoveryUtil; +import org.joda.time.DateTime; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +/** + * Steps + * 1) Submit a RealtimeIndexTask + * 2) Load Data using EventReceiverFirehose + * 3) Runs queries and verifies that the ingested data is available for queries + * 4) Waits for handover of the segment to historical node + * 5) Queries data from historical node and verifies handover + * 6) Removes and Delete the created Data Segment + */ +@Guice(moduleFactory = DruidTestModuleFactory.class) +public class ITRealtimeIndexTaskTest extends AbstractIndexerTest +{ + private static final Logger LOG = new Logger(ITRealtimeIndexTaskTest.class); + private static final String REALTIME_TASK_RESOURCE = "/indexer/wikipedia_realtime_index_task.json"; + private static final String EVENT_RECEIVER_SERVICE_NAME = "eventReceiverServiceName"; + private static final String EVENT_DATA_FILE = "/indexer/wikipedia_index_data.json"; + private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_index_queries.json"; + private static final String INDEX_DATASOURCE = "wikipedia_index_test"; + @Inject + ServerDiscoveryFactory factory; + @Inject + @Global + HttpClient httpClient; + + @Inject + IntegrationTestingConfig config; + + @Test + public void testRealtimeIndexTask() throws Exception + { + try { + // the task will run for 3 minutes and then shutdown itself + String task = setShutOffTime( + getTaskAsString(REALTIME_TASK_RESOURCE), + new DateTime(System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(2)) + ); + String taskID = indexer.submitTask(task); + postEvents(); + + // sleep for a while to let the events ingested + TimeUnit.SECONDS.sleep(5); + + // should hit the queries on realtime task + this.queryHelper.testQueriesFromFile(INDEX_QUERIES_RESOURCE, 2); + // wait for the task to complete + indexer.waitUntilTaskCompletes(taskID); + + // task should complete only after the segments are loaded by historical node + RetryUtil.retryUntil( + new Callable() + { + @Override + public Boolean call() throws Exception + { + return coordinator.areSegmentsLoaded(INDEX_DATASOURCE); + } + }, + true, + 60000, + 10, + "Real-time generated segments loaded" + ); + + // run queries on historical nodes + this.queryHelper.testQueriesFromFile(INDEX_QUERIES_RESOURCE, 2); + } + catch (Exception e) { + e.printStackTrace(); + Throwables.propagate(e); + } + finally { + unloadAndKillData(INDEX_DATASOURCE); + } + } + + private String setShutOffTime(String taskAsString, DateTime time) + { + return taskAsString.replace("#SHUTOFFTIME", time.toString()); + } + + public void postEvents() throws Exception + { + final ServerDiscoverySelector eventReceiverSelector = factory.createSelector(EVENT_RECEIVER_SERVICE_NAME); + eventReceiverSelector.start(); + try { + ServerDiscoveryUtil.waitUntilInstanceReady(eventReceiverSelector, "Event Receiver"); + // Access the docker VM mapped host and port instead of service announced in zookeeper + String host = config.getMiddleManagerHost() + ":" + eventReceiverSelector.pick().getPort(); + LOG.info("Event Receiver Found at host %s", host); + EventReceiverFirehoseTestClient client = new EventReceiverFirehoseTestClient( + host, + EVENT_RECEIVER_SERVICE_NAME, + jsonMapper, + httpClient + ); + client.postEventsFromFile(EVENT_DATA_FILE); + } + finally { + eventReceiverSelector.stop(); + } + } +} diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java new file mode 100644 index 00000000000..7e7c87f9d4e --- /dev/null +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java @@ -0,0 +1,172 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.tests.indexer; + +import com.beust.jcommander.internal.Lists; +import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.metamx.common.logger.Logger; +import com.metamx.http.client.HttpClient; +import io.druid.curator.discovery.ServerDiscoveryFactory; +import io.druid.curator.discovery.ServerDiscoverySelector; +import io.druid.guice.annotations.Global; +import io.druid.testing.IntegrationTestingConfig; +import io.druid.testing.clients.EventReceiverFirehoseTestClient; +import io.druid.testing.guice.DruidTestModuleFactory; +import io.druid.testing.utils.RetryUtil; +import io.druid.testing.utils.ServerDiscoveryUtil; +import org.joda.time.DateTime; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +@Guice(moduleFactory = DruidTestModuleFactory.class) +public class ITUnionQueryTest extends AbstractIndexerTest +{ + private static final Logger LOG = new Logger(ITUnionQueryTest.class); + private static final String REALTIME_TASK_RESOURCE = "/indexer/wikipedia_realtime_index_task.json"; + private static final String EVENT_RECEIVER_SERVICE_PREFIX = "eventReceiverServiceName"; + private static final String UNION_DATA_FILE = "/indexer/wikipedia_index_data.json"; + private static final String UNION_QUERIES_RESOURCE = "/indexer/union_queries.json"; + private static final String UNION_DATASOURCE = "wikipedia_index_test"; + + @Inject + ServerDiscoveryFactory factory; + + @Inject + @Global + HttpClient httpClient; + + @Inject + IntegrationTestingConfig config; + + @Test + public void testRealtimeIndexTask() throws Exception + { + final int numTasks = 4; + + try { + // Load 4 datasources with same dimensions + String task = setShutOffTime( + getTaskAsString(REALTIME_TASK_RESOURCE), + new DateTime(System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(3)) + ); + List taskIDs = Lists.newArrayList(); + for (int i = 0; i < numTasks; i++) { + taskIDs.add( + indexer.submitTask( + withServiceName( + withDataSource(task, UNION_DATASOURCE + i), + EVENT_RECEIVER_SERVICE_PREFIX + i + ) + ) + ); + } + for (int i = 0; i < numTasks; i++) { + postEvents(i); + } + + // sleep for a while to let the events ingested + TimeUnit.SECONDS.sleep(5); + + // should hit the queries on realtime task + LOG.info("Running Union Queries.."); + this.queryHelper.testQueriesFromFile(UNION_QUERIES_RESOURCE, 2); + + // wait for the task to complete + for (int i = 0; i < numTasks; i++) { + indexer.waitUntilTaskCompletes(taskIDs.get(i)); + } + // task should complete only after the segments are loaded by historical node + for (int i = 0; i < numTasks; i++) { + final int taskNum = i; + RetryUtil.retryUntil( + new Callable() + { + @Override + public Boolean call() throws Exception + { + return coordinator.areSegmentsLoaded(UNION_DATASOURCE + taskNum); + } + }, + true, + 60000, + 10, + "Real-time generated segments loaded" + ); + } + // run queries on historical nodes + this.queryHelper.testQueriesFromFile(UNION_QUERIES_RESOURCE, 2); + + } + catch (Exception e) { + e.printStackTrace(); + throw Throwables.propagate(e); + } + finally { + for (int i = 0; i < numTasks; i++) { + unloadAndKillData(UNION_DATASOURCE + i); + } + } + + } + + private String setShutOffTime(String taskAsString, DateTime time) + { + return taskAsString.replace("#SHUTOFFTIME", time.toString()); + } + + private String withDataSource(String taskAsString, String dataSource) + { + return taskAsString.replace(UNION_DATASOURCE, dataSource); + } + + private String withServiceName(String taskAsString, String serviceName) + { + return taskAsString.replace(EVENT_RECEIVER_SERVICE_PREFIX, serviceName); + } + + public void postEvents(int id) throws Exception + { + final ServerDiscoverySelector eventReceiverSelector = factory.createSelector(EVENT_RECEIVER_SERVICE_PREFIX + id); + eventReceiverSelector.start(); + try { + ServerDiscoveryUtil.waitUntilInstanceReady(eventReceiverSelector, "Event Receiver"); + // Access the docker VM mapped host and port instead of service announced in zookeeper + String host = config.getMiddleManagerHost() + ":" + eventReceiverSelector.pick().getPort(); + + LOG.info("Event Receiver Found at host [%s]", host); + + EventReceiverFirehoseTestClient client = new EventReceiverFirehoseTestClient( + host, + EVENT_RECEIVER_SERVICE_PREFIX + id, + jsonMapper, + httpClient + ); + client.postEventsFromFile(UNION_DATA_FILE); + } + finally { + eventReceiverSelector.stop(); + } + } +} diff --git a/integration-tests/src/test/java/io/druid/tests/query/ITTwitterQueryTest.java b/integration-tests/src/test/java/io/druid/tests/query/ITTwitterQueryTest.java new file mode 100644 index 00000000000..584efdcf01b --- /dev/null +++ b/integration-tests/src/test/java/io/druid/tests/query/ITTwitterQueryTest.java @@ -0,0 +1,65 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.tests.query; + +import com.google.inject.Inject; +import io.druid.testing.clients.CoordinatorResourceTestClient; +import io.druid.testing.guice.DruidTestModuleFactory; +import io.druid.testing.utils.FromFileTestQueryHelper; +import io.druid.testing.utils.RetryUtil; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.util.concurrent.Callable; + +@Guice(moduleFactory = DruidTestModuleFactory.class) +public class ITTwitterQueryTest +{ + private static final String TWITTER_DATA_SOURCE = "twitterstream"; + private static final String TWITTER_QUERIES_RESOURCE = "/queries/twitterstream_queries.json"; + @Inject + CoordinatorResourceTestClient coordinatorClient; + @Inject + private FromFileTestQueryHelper queryHelper; + + @BeforeMethod + public void before() + { + // ensure that the segments twitter segments are loaded completely + RetryUtil.retryUntilTrue( + new Callable() + { + @Override + public Boolean call() throws Exception + { + return coordinatorClient.areSegmentsLoaded(TWITTER_DATA_SOURCE); + } + }, "twitter segment load" + ); + } + + @Test + public void testQueriesFromFile() throws Exception + { + queryHelper.testQueriesFromFile(TWITTER_QUERIES_RESOURCE, 2); + } + +} diff --git a/integration-tests/src/test/java/io/druid/tests/query/ITWikipediaQueryTest.java b/integration-tests/src/test/java/io/druid/tests/query/ITWikipediaQueryTest.java new file mode 100644 index 00000000000..6279ca78252 --- /dev/null +++ b/integration-tests/src/test/java/io/druid/tests/query/ITWikipediaQueryTest.java @@ -0,0 +1,65 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.tests.query; + +import com.google.inject.Inject; +import io.druid.testing.clients.CoordinatorResourceTestClient; +import io.druid.testing.guice.DruidTestModuleFactory; +import io.druid.testing.utils.FromFileTestQueryHelper; +import io.druid.testing.utils.RetryUtil; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.util.concurrent.Callable; + +@Guice(moduleFactory = DruidTestModuleFactory.class) +public class ITWikipediaQueryTest +{ + private static final String WIKIPEDIA_DATA_SOURCE = "wikipedia_editstream"; + private static final String WIKIPEDIA_QUERIES_RESOURCE = "/queries/wikipedia_editstream_queries.json"; + @Inject + private CoordinatorResourceTestClient coordinatorClient; + @Inject + private FromFileTestQueryHelper queryHelper; + + @BeforeMethod + public void before() + { + // ensure that twitter segments are loaded completely + RetryUtil.retryUntilTrue( + new Callable() + { + @Override + public Boolean call() throws Exception + { + return coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE); + } + }, "wikipedia segment load" + ); + } + + @Test + public void testQueriesFromFile() throws Exception + { + queryHelper.testQueriesFromFile(WIKIPEDIA_QUERIES_RESOURCE, 2); + } + +} diff --git a/integration-tests/src/test/resources/indexer/select.query b/integration-tests/src/test/resources/indexer/select.query new file mode 100644 index 00000000000..463ffef08e1 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/select.query @@ -0,0 +1,19 @@ +{ + "queryType": "select", + "intervals": ["2013-08-31/2013-09-01"], + "dataSource": "wikipedia_index_test", + "granularity": "all", + "filter": { + "type": "selector", + "dimension": "language", + "value": "en" + }, + "pagingSpec": { + "threshold": 10 + }, + "context": { + "useCache": "false", + "populateCache": "false", + "timeout": 60000 + } + } diff --git a/integration-tests/src/test/resources/indexer/select_reindex.query b/integration-tests/src/test/resources/indexer/select_reindex.query new file mode 100644 index 00000000000..e6a162377dc --- /dev/null +++ b/integration-tests/src/test/resources/indexer/select_reindex.query @@ -0,0 +1,19 @@ +{ + "queryType": "select", + "intervals": ["2013-08-31/2013-09-01"], + "dataSource": "wikipedia_reindex_test", + "granularity": "all", + "filter": { + "type": "selector", + "dimension": "language", + "value": "en" + }, + "pagingSpec": { + "threshold": 10 + }, + "context": { + "useCache": "false", + "populateCache": "false", + "timeout": 60000 + } + } diff --git a/integration-tests/src/test/resources/indexer/union_queries.json b/integration-tests/src/test/resources/indexer/union_queries.json new file mode 100644 index 00000000000..9c0ce5a0de5 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/union_queries.json @@ -0,0 +1,564 @@ +[ + { + "description": "timeseries, filtered, all aggs, all", + "query": { + "queryType": "timeseries", + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test1", "wikipedia_index_test2", "wikipedia_index_test3", + "wikipedia_index_test0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "filter": { + "type": "selector", + "dimension": "language", + "value": "en" + }, + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + }, + { + "type": "doubleSum", + "fieldName": "delta", + "name": "delta" + } + ], + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T01:02:33.000Z", + "result": { + "added": 2064.0, + "count": 8, + "delta": 748.0, + "deleted": 1316.0, + "rows": 8 + } + } + ] + }, + { + "description": "topN, all aggs, page dim, uniques metric", + "query": { + "queryType": "topN", + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test1", "wikipedia_index_test2", "wikipedia_index_test3", + "wikipedia_index_test0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + }, + { + "type": "doubleSum", + "fieldName": "delta", + "name": "delta" + } + ], + "dimension": "page", + "metric": "added", + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T01:02:33.000Z", + "result": [ + { + "added": 3620.0, + "count": 4, + "page": "Crimson Typhoon", + "delta": 3600.0, + "deleted": 20.0, + "rows": 4 + }, + { + "added": 1836.0, + "count": 4, + "page": "Striker Eureka", + "delta": 1320.0, + "deleted": 516.0, + "rows": 4 + }, + { + "added": 492.0, + "count": 4, + "page": "Cherno Alpha", + "delta": 444.0, + "deleted": 48.0, + "rows": 4 + } + ] + } + ] + }, + { + "description": "topN, all aggs, page dim, count metric, postAggs", + "query": { + "queryType": "topN", + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test1", "wikipedia_index_test2", "wikipedia_index_test3", + "wikipedia_index_test0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + }, + { + "type": "doubleSum", + "fieldName": "delta", + "name": "delta" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfAddedDeletedConst", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "added", + "fieldName": "added" + }, + { + "type": "arithmetic", + "name": "", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "constant", + "name": "constant", + "value": 1000 + } + ] + } + ] + } + ], + "dimension": "page", + "metric": "added", + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T01:02:33.000Z", + "result": [ + { + "added": 3620.0, + "count": 4, + "page": "Crimson Typhoon", + "delta": 3600.0, + "deleted": 20.0, + "sumOfAddedDeletedConst": 4640.0, + "rows": 4 + }, + { + "added": 1836.0, + "count": 4, + "page": "Striker Eureka", + "delta": 1320.0, + "deleted": 516.0, + "sumOfAddedDeletedConst": 3352.0, + "rows": 4 + }, + { + "added": 492.0, + "count": 4, + "page": "Cherno Alpha", + "delta": 444.0, + "deleted": 48.0, + "sumOfAddedDeletedConst": 1540.0, + "rows": 4 + } + ] + } + ] + }, + { + "description": "topN, lexicographic, two aggs, language dim, postAggs", + "query": { + "queryType": "topN", + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test1", "wikipedia_index_test2", "wikipedia_index_test3", + "wikipedia_index_test0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfRowsAndCount", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "rows", + "fieldName": "rows" + }, + { + "type": "fieldAccess", + "name": "count", + "fieldName": "count" + } + ] + } + ], + "dimension": "language", + "metric": { + "type": "lexicographic", + "previousStop": "a" + }, + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T01:02:33.000Z", + "result": [ + { + "sumOfRowsAndCount": 16.0, + "count": 8, + "language": "en", + "rows": 8 + }, + { + "sumOfRowsAndCount": 8.0, + "count": 4, + "language": "ja", + "rows": 4 + }, + { + "sumOfRowsAndCount": 8.0, + "count": 4, + "language": "ru", + "rows": 4 + } + ] + } + ] + }, + { + "description": "groupBy, two aggs, namespace dim, postAggs", + "query": { + "queryType": "groupBy", + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test1", "wikipedia_index_test2", "wikipedia_index_test3", + "wikipedia_index_test0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfRowsAndCount", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "rows", + "fieldName": "rows" + }, + { + "type": "fieldAccess", + "name": "count", + "fieldName": "count" + } + ] + } + ], + "dimensions": ["namespace"], + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "version": "v1", + "timestamp": "2013-08-31T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 16.0, + "count": 8, + "rows": 8, + "namespace": "article" + } + }, + { + "version": "v1", + "timestamp": "2013-08-31T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 24.0, + "count": 12, + "rows": 12, + "namespace": "wikipedia" + } + } + ] + }, + { + "description": "groupBy, two aggs, namespace + robot dim, postAggs", + "query": { + "queryType": "groupBy", + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test1", "wikipedia_index_test2", "wikipedia_index_test3", + "wikipedia_index_test0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfRowsAndCount", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "rows", + "fieldName": "rows" + }, + { + "type": "fieldAccess", + "name": "count", + "fieldName": "count" + } + ] + } + ], + "dimensions": ["namespace", "robot"], + "limitSpec": { + "type": "default", + "limit": 3, + "orderBy": ["robot", "namespace"] + }, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "version": "v1", + "timestamp": "2013-08-31T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 8.0, + "count": 4, + "robot": "false", + "rows": 4, + "namespace": "article" + } + }, + { + "version": "v1", + "timestamp": "2013-08-31T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 8.0, + "count": 4, + "robot": "true", + "rows": 4, + "namespace": "article" + } + }, + { + "version": "v1", + "timestamp": "2013-08-31T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 24.0, + "count": 12, + "robot": "true", + "rows": 12, + "namespace": "wikipedia" + } + } + ] + }, + { + "query": { + "queryType": "search", + "intervals": ["2013-08-31/2013-09-01"], + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test1", "wikipedia_index_test2", "wikipedia_index_test3", + "wikipedia_index_test0" + ] + }, + "granularity": "all", + "query": { + "type": "insensitive_contains", + "value": "ip" + }, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T00:00:00.000Z", + "result": [ + { + "dimension": "user", + "value": "triplets" + }, + { + "dimension": "namespace", + "value": "wikipedia" + } + ] + } + ] + }, + { + "description": "timeboundary, 1 agg, union", + "query": { + "queryType": "timeBoundary", + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test1", "wikipedia_index_test2", "wikipedia_index_test3", + "wikipedia_index_test0" + ] + } + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T01:02:33.000Z", + "result": { + "minTime": "2013-08-31T01:02:33.000Z", + "maxTime": "2013-08-31T12:41:27.000Z" + } + } + ] + } +] diff --git a/integration-tests/src/test/resources/indexer/union_select_query.json b/integration-tests/src/test/resources/indexer/union_select_query.json new file mode 100644 index 00000000000..76e4d0be63e --- /dev/null +++ b/integration-tests/src/test/resources/indexer/union_select_query.json @@ -0,0 +1,121 @@ +{ + "query": { + "queryType": "select", + "intervals": ["2013-08-31/2013-09-01"], + "dataSource": { + "type": "union", + "dataSources": [ + "wikipedia_index_test" + ] + }, + "granularity": "all", + "filter": { + "type": "selector", + "dimension": "language", + "value": "en" + }, + "pagingSpec": { + "threshold": 10 + }, + "context": { + "useCache": "false", + "populateCache": "false", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T01:02:33.000Z", + "result": { + "pagingIdentifiers": { + "wikipedia_index_test0_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:43.993Z": 0, + "wikipedia_index_test1_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.108Z": 0, + "wikipedia_index_test2_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.236Z": 0, + "wikipedia_index_test3_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.374Z": 0 + }, + "events": [ + { + "segmentId": "wikipedia_index_test0_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:43.993Z", + "offset": 0, + "event": { + "timestamp": "2013-08-31T01:02:33.000Z", + "page": "Gypsy Danger", + "added": 57.0, + "deleted": 200.0 + } + }, + { + "segmentId": "wikipedia_index_test1_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.108Z", + "offset": 0, + "event": { + "timestamp": "2013-08-31T01:02:33.000Z", + "page": "Gypsy Danger", + "added": 57.0, + "deleted": 200.0 + } + }, + { + "segmentId": "wikipedia_index_test2_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.236Z", + "offset": 0, + "event": { + "timestamp": "2013-08-31T01:02:33.000Z", + "page": "Gypsy Danger", + "added": 57.0, + "deleted": 200.0 + } + }, + { + "segmentId": "wikipedia_index_test3_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.374Z", + "offset": 0, + "event": { + "timestamp": "2013-08-31T01:02:33.000Z", + "page": "Gypsy Danger", + "added": 57.0, + "deleted": 200.0 + } + }, + { + "segmentId": "wikipedia_index_test0_2013-08-31T0com.metamx.common.ISE: one or more twitter queries failed0:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:43.993Z", + "offset": 0, + "event": { + "timestamp": "2013-08-31T03:32:45.000Z", + "page": "Striker Eureka", + "added": 459.0, + "deleted": 129.0 + } + }, + { + "segmentId": "wikipedia_index_test1_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.108Z", + "offset": 0, + "event": { + "timestamp": "2013-08-31T03:32:45.000Z", + "page": "Striker Eureka", + "added": 459.0, + "deleted": 129.0 + } + }, + { + "segmentId": "wikipedia_index_test2_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.236Z", + "offset": 0, + "event": { + "timestamp": "2013-08-31T03:32:45.000Z", + "page": "Striker Eureka", + "added": 459.0, + "deleted": 129.0 + } + }, + { + "segmentId": "wikipedia_index_test3_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.374Z", + "offset": 0, + "event": { + "timestamp": "2013-08-31T03:32:45.000Z", + "page": "Striker Eureka", + "added": 459.0, + "deleted": 129.0 + } + } + ] + } + } + ] +} diff --git a/integration-tests/src/test/resources/indexer/wikipedia_index_data.json b/integration-tests/src/test/resources/indexer/wikipedia_index_data.json new file mode 100644 index 00000000000..592996e1805 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/wikipedia_index_data.json @@ -0,0 +1,5 @@ +{"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143} +{"timestamp": "2013-08-31T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330} +{"timestamp": "2013-08-31T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111} +{"timestamp": "2013-08-31T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900} +{"timestamp": "2013-08-31T12:41:27Z", "page": "Coyote Tango", "language" : "ja", "user" : "stringer", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo", "added": 1, "deleted": 10, "delta": -9} \ No newline at end of file diff --git a/integration-tests/src/test/resources/indexer/wikipedia_index_queries.json b/integration-tests/src/test/resources/indexer/wikipedia_index_queries.json new file mode 100644 index 00000000000..caea782fcb5 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/wikipedia_index_queries.json @@ -0,0 +1,16 @@ +[ + { + "description": "timeseries, 1 agg, all", + "query":{ + "queryType" : "timeBoundary", + "dataSource": "wikipedia_index_test" + }, + "expectedResults":[ { + "timestamp" : "2013-08-31T01:02:33.000Z", + "result" : { + "minTime" : "2013-08-31T01:02:33.000Z", + "maxTime" : "2013-08-31T12:41:27.000Z" + } + } ] + } +] \ No newline at end of file diff --git a/integration-tests/src/test/resources/indexer/wikipedia_index_task.json b/integration-tests/src/test/resources/indexer/wikipedia_index_task.json new file mode 100644 index 00000000000..369f9aac6b4 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/wikipedia_index_task.json @@ -0,0 +1,59 @@ +{ + "type": "index", + "spec": { + "dataSchema": { + "dataSource": "wikipedia_index_test", + "metricsSpec": [ + { + "type": "count", + "name": "count" + }, + { + "type": "doubleSum", + "name": "added", + "fieldName": "added" + }, + { + "type": "doubleSum", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "doubleSum", + "name": "delta", + "fieldName": "delta" + } + ], + "granularitySpec": { + "segmentGranularity": "DAY", + "queryGranularity": "second", + "intervals" : [ "2013-08-31/2013-09-01" ] + }, + "parser": { + "parseSpec": { + "format" : "json", + "timestampSpec": { + "column": "timestamp" + }, + "dimensionsSpec": { + "dimensions": [ + "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", + "namespace", "continent", "country", "region", "city" + ] + } + } + } + }, + "ioConfig": { + "type": "index", + "firehose": { + "type": "local", + "baseDir": "/resources/indexer", + "filter": "wikipedia_index_data.json" + } + }, + "tuningConfig": { + "type": "index" + } + } +} \ No newline at end of file diff --git a/integration-tests/src/test/resources/indexer/wikipedia_realtime_index_task.json b/integration-tests/src/test/resources/indexer/wikipedia_realtime_index_task.json new file mode 100644 index 00000000000..a204abd9512 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/wikipedia_realtime_index_task.json @@ -0,0 +1,71 @@ +{ + "type": "index_realtime", + "spec": { + "dataSchema": { + "dataSource": "wikipedia_index_test", + "metricsSpec": [ + { + "type": "count", + "name": "count" + }, + { + "type": "doubleSum", + "name": "added", + "fieldName": "added" + }, + { + "type": "doubleSum", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "doubleSum", + "name": "delta", + "fieldName": "delta" + } + ], + "granularitySpec": { + "segmentGranularity": "DAY", + "queryGranularity": "second" + }, + "parser": { + "type" : "map", + "parseSpec": { + "timestampSpec": { + "column": "timestamp", + "format": "iso" + }, + "dimensionsSpec" : { + "dimensions": [ + "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", + "namespace", "continent", "country", "region", "city" + ] + } + } + } + }, + "ioConfig": { + "type": "realtime", + "firehose": { + "type": "timed", + "shutoffTime": "#SHUTOFFTIME", + "delegate": { + "type": "receiver", + "serviceName": "eventReceiverServiceName", + "bufferSize": 100000 + } + } + + }, + "tuningConfig": { + "type": "realtime", + "maxRowsInMemory": 1, + "intermediatePersistPeriod": "PT1M", + "windowPeriod": "PT1M", + "rejectionPolicy": { + "type": "none" + } + } + } + +} diff --git a/integration-tests/src/test/resources/indexer/wikipedia_reindex_task.json b/integration-tests/src/test/resources/indexer/wikipedia_reindex_task.json new file mode 100644 index 00000000000..7d4b937dfd9 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/wikipedia_reindex_task.json @@ -0,0 +1,62 @@ +{ + "type": "index", + "spec": { + "dataSchema": { + "dataSource": "wikipedia_index_test", + "metricsSpec": [ + { + "type": "count", + "name": "count" + }, + { + "type": "doubleSum", + "name": "added", + "fieldName": "added" + }, + { + "type": "doubleSum", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "doubleSum", + "name": "delta", + "fieldName": "delta" + } + ], + "granularitySpec": { + "segmentGranularity": "DAY", + "queryGranularity": "second", + "intervals" : [ "2013-08-31/2013-09-01" ] + }, + "parser": { + "parseSpec": { + "format" : "json", + "timestampSpec": { + "column": "timestamp", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", + "namespace", "continent", "country", "region", "city" + ] + } + } + } + }, + "ioConfig": { + "type": "index", + "firehose": { + "type": "ingestSegment", + "dataSource": "wikipedia_index_test", + "dimensions": ["user", "nonexist"], + "metrics": ["added", "added2"], + "interval": "2013-08-31/2013-09-01" + } + }, + "tuningConfig": { + "type": "index" + } + } +} diff --git a/integration-tests/src/test/resources/queries/twitterstream_queries.json b/integration-tests/src/test/resources/queries/twitterstream_queries.json new file mode 100644 index 00000000000..78bb134ef5c --- /dev/null +++ b/integration-tests/src/test/resources/queries/twitterstream_queries.json @@ -0,0 +1,780 @@ +[ + { + "description": "timeseries, 2 aggs", + "query": { + "queryType": "timeseries", + "dataSource": "twitterstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-04T00:00:00.000"], + "granularity": "day", + "aggregations": [ + { + "type": "doubleSum", + "name": "num_tweets", + "fieldName": "count" + }, + { + "type": "doubleSum", + "name": "tweet_length", + "fieldName": "tweet_length" + } + ], + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": { + "tweet_length": 2.40241323E8, + "num_tweets": 3754028.0 + } + }, + { + "timestamp": "2013-01-02T00:00:00.000Z", + "result": { + "tweet_length": 2.46397801E8, + "num_tweets": 3799466.0 + } + }, + { + "timestamp": "2013-01-03T00:00:00.000Z", + "result": { + "tweet_length": 2.31365019E8, + "num_tweets": 3552419.0 + } + } + ] + }, + { + "description": "topN, 2 aggs, lexicographic", + "query": { + "queryType": "topN", + "dataSource": "twitterstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-04T00:00:00.000"], + "granularity": "day", + "aggregations": [ + { + "type": "doubleSum", + "name": "num_tweets", + "fieldName": "count" + }, + { + "type": "doubleSum", + "name": "tweet_length", + "fieldName": "tweet_length" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "avg_tweet_len", + "fn": "/", + "fields": [ + { + "type": "fieldAccess", + "name": "tweet_length", + "fieldName": "tweet_length" + }, + { + "type": "fieldAccess", + "name": "num_tweets", + "fieldName": "num_tweets" + } + ] + } + ], + "dimension": "user_name", + "metric": { + "type": "lexicographic" + }, + "threshold": 2, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "user_name": "000000000000087", + "tweet_length": 14.0, + "num_tweets": 1.0, + "avg_tweet_len": 14.0 + }, + { + "user_name": "0000000000mghi", + "tweet_length": 291.0, + "num_tweets": 4.0, + "avg_tweet_len": 72.75 + } + ] + }, + { + "timestamp": "2013-01-02T00:00:00.000Z", + "result": [ + { + "user_name": "000000000037", + "tweet_length": 13.0, + "num_tweets": 1.0, + "avg_tweet_len": 13.0 + }, + { + "user_name": "0000000000mghi", + "tweet_length": 21.0, + "num_tweets": 1.0, + "avg_tweet_len": 21.0 + } + ] + }, + { + "timestamp": "2013-01-03T00:00:00.000Z", + "result": [ + { + "user_name": "000000007", + "tweet_length": 37.0, + "num_tweets": 1.0, + "avg_tweet_len": 37.0 + }, + { + "user_name": "00000000b", + "tweet_length": 119.0, + "num_tweets": 1.0, + "avg_tweet_len": 119.0 + } + ] + } + ] + }, + { + "description": "topN, 2 aggs", + "query": { + "queryType": "topN", + "dataSource": "twitterstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-04T00:00:00.000"], + "granularity": "day", + "aggregations": [ + { + "type": "doubleSum", + "name": "num_tweets", + "fieldName": "count" + }, + { + "type": "doubleSum", + "name": "tweet_length", + "fieldName": "tweet_length" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "avg_tweet_len", + "fn": "/", + "fields": [ + { + "type": "fieldAccess", + "name": "tweet_length", + "fieldName": "tweet_length" + }, + { + "type": "fieldAccess", + "name": "num_tweets", + "fieldName": "num_tweets" + } + ] + } + ], + "dimension": "user_name", + "metric": { + "type": "numeric", + "metric": "num_tweets" + }, + "threshold": 2, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "user_name": "Favstar_Bot", + "tweet_length": 2002.0, + "num_tweets": 33.0, + "avg_tweet_len": 60.666666666666664 + }, + { + "user_name": "SportsAB", + "tweet_length": 1114.0, + "num_tweets": 26.0, + "avg_tweet_len": 42.84615384615385 + } + ] + }, + { + "timestamp": "2013-01-02T00:00:00.000Z", + "result": [ + { + "user_name": "Favstar_Bot", + "tweet_length": 2185.0, + "num_tweets": 36.0, + "avg_tweet_len": 60.69444444444444 + }, + { + "user_name": "SportsAB", + "tweet_length": 1148.0, + "num_tweets": 23.0, + "avg_tweet_len": 49.91304347826087 + } + ] + }, + { + "timestamp": "2013-01-03T00:00:00.000Z", + "result": [ + { + "user_name": "SportsAB", + "tweet_length": 882.0, + "num_tweets": 22.0, + "avg_tweet_len": 40.09090909090909 + }, + { + "user_name": "furin0620", + "tweet_length": 867.0, + "num_tweets": 21.0, + "avg_tweet_len": 41.285714285714285 + } + ] + } + ] + }, + { + "description": "topN, 2 aggs, filtered", + "query": { + "queryType": "topN", + "dataSource": "twitterstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-04T00:00:00.000"], + "granularity": "day", + "filter": { + "type": "or", + "fields": [ + { + "type": "selector", + "dimension": "user_name", + "value": "Favstar_Bot" + }, + { + "type": "selector", + "dimension": "user_name", + "value": "SportsAB" + }, + { + "type": "selector", + "dimension": "user_name", + "value": "furin0620" + } + ] + }, + "aggregations": [ + { + "type": "doubleSum", + "name": "num_tweets", + "fieldName": "count" + }, + { + "type": "doubleSum", + "name": "tweet_length", + "fieldName": "tweet_length" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "avg_tweet_len", + "fn": "/", + "fields": [ + { + "type": "fieldAccess", + "name": "tweet_length", + "fieldName": "tweet_length" + }, + { + "type": "fieldAccess", + "name": "num_tweets", + "fieldName": "num_tweets" + } + ] + } + ], + "dimension": "user_name", + "metric": { + "type": "numeric", + "metric": "num_tweets" + }, + "threshold": 2, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "user_name": "Favstar_Bot", + "tweet_length": 2002.0, + "num_tweets": 33.0, + "avg_tweet_len": 60.666666666666664 + }, + { + "user_name": "SportsAB", + "tweet_length": 1114.0, + "num_tweets": 26.0, + "avg_tweet_len": 42.84615384615385 + } + ] + }, + { + "timestamp": "2013-01-02T00:00:00.000Z", + "result": [ + { + "user_name": "Favstar_Bot", + "tweet_length": 2185.0, + "num_tweets": 36.0, + "avg_tweet_len": 60.69444444444444 + }, + { + "user_name": "SportsAB", + "tweet_length": 1148.0, + "num_tweets": 23.0, + "avg_tweet_len": 49.91304347826087 + } + ] + }, + { + "timestamp": "2013-01-03T00:00:00.000Z", + "result": [ + { + "user_name": "SportsAB", + "tweet_length": 882.0, + "num_tweets": 22.0, + "avg_tweet_len": 40.09090909090909 + }, + { + "user_name": "furin0620", + "tweet_length": 867.0, + "num_tweets": 21.0, + "avg_tweet_len": 41.285714285714285 + } + ] + } + ] + }, + { + "description": "groupBy", + "query": { + "queryType": "groupBy", + "dataSource": "twitterstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-04T00:00:00.000"], + "granularity": "day", + "aggregations": [ + { + "type": "doubleSum", + "name": "num_tweets", + "fieldName": "count" + }, + { + "type": "doubleSum", + "name": "tweet_length", + "fieldName": "tweet_length" + } + ], + "dimensions": ["has_links"] + }, + "expectedResults": [ + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "has_links": "No", + "tweet_length": 2.08803904E8, + "num_tweets": 3377791.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "has_links": "Yes", + "tweet_length": 3.143742E7, + "num_tweets": 376237.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-02T00:00:00.000Z", + "event": { + "has_links": "No", + "tweet_length": 2.10402688E8, + "num_tweets": 3375243.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-02T00:00:00.000Z", + "event": { + "has_links": "Yes", + "tweet_length": 3.599512E7, + "num_tweets": 424223.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-03T00:00:00.000Z", + "event": { + "has_links": "No", + "tweet_length": 1.96451456E8, + "num_tweets": 3144985.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-03T00:00:00.000Z", + "event": { + "has_links": "Yes", + "tweet_length": 3.4913568E7, + "num_tweets": 407434.0 + } + } + ] + }, + { + "query": { + "queryType": "search", + "intervals": ["2013-01-01T00:00:00.000/2013-01-04T00:00:00.000"], + "dataSource": "twitterstream", + "granularity": "all", + "searchDimensions": ["user_name"], + "sort": { + "type": "lexicographic" + }, + "query": { + "type": "insensitive_contains", + "value": "Sports" + }, + "limit": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "dimension": "user_name", + "value": "1011Sports" + }, + { + "dimension": "user_name", + "value": "11AliveSports" + }, + { + "dimension": "user_name", + "value": "1World_Sports" + } + ] + } + ] + }, + { + "description": "groupByArbitraryInterval", + "query": { + "queryType": "groupBy", + "dataSource": "twitterstream", + "intervals": ["2013-01-01T15:10:10.090/2013-01-03T19:30:01.090"], + "granularity": "day", + "aggregations": [ + { + "type": "doubleSum", + "name": "num_tweets", + "fieldName": "count" + }, + { + "type": "doubleSum", + "name": "tweet_length", + "fieldName": "tweet_length" + } + ], + "dimensions": ["has_links"] + }, + "expectedResults": [ + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "has_links": "No", + "tweet_length": 7.4820448E7, + "num_tweets": 1170229.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "has_links": "Yes", + "tweet_length": 1.149719E7, + "num_tweets": 136582.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-02T00:00:00.000Z", + "event": { + "has_links": "No", + "tweet_length": 2.10402688E8, + "num_tweets": 3375243.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-02T00:00:00.000Z", + "event": { + "has_links": "Yes", + "tweet_length": 3.599512E7, + "num_tweets": 424223.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-03T00:00:00.000Z", + "event": { + "has_links": "No", + "tweet_length": 1.59141088E8, + "num_tweets": 2567986.0 + } + }, + { + "version": "v1", + "timestamp": "2013-01-03T00:00:00.000Z", + "event": { + "has_links": "Yes", + "tweet_length": 2.8345444E7, + "num_tweets": 328917.0 + } + } + ] + }, + { + "description": "segmentMetadata", + "query": { + "queryType": "segmentMetadata", + "dataSource": "twitterstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-04T00:00:00.000"], + "toInclude": { + "type": "list", + "columns": ["has_links", "has_links"] + } + }, + "expectedResults": [ + { + "id": "twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9", + "intervals": ["2013-01-01T00:00:00.000Z/2013-01-02T00:00:00.000Z"], + "columns": { + "has_links": { + "type": "STRING", + "size": 7773438, + "cardinality": 2, + "errorMessage": null + } + }, + "size": 747056474 + }, + { + "id": "twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9", + "intervals": ["2013-01-02T00:00:00.000Z/2013-01-03T00:00:00.000Z"], + "columns": { + "has_links": { + "type": "STRING", + "size": 7901000, + "cardinality": 2, + "errorMessage": null + } + }, + "size": 755796690 + }, + { + "id": "twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9", + "intervals": ["2013-01-03T00:00:00.000Z/2013-01-04T00:00:00.000Z"], + "columns": { + "has_links": { + "type": "STRING", + "size": 7405654, + "cardinality": 2, + "errorMessage": null + } + }, + "size": 706893542 + } + ] + }, + { + "description": "topN, 2 aggs, topN over dependent postAgg", + "query": { + "queryType": "topN", + "dataSource": "twitterstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-04T00:00:00.000"], + "granularity": "day", + "aggregations": [ + { + "type": "doubleSum", + "name": "num_tweets", + "fieldName": "count" + }, + { + "type": "doubleSum", + "name": "tweet_length", + "fieldName": "tweet_length" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "avg_tweet_len", + "fn": "/", + "fields": [ + { + "type": "fieldAccess", + "name": "tweet_length", + "fieldName": "tweet_length" + }, + { + "type": "fieldAccess", + "name": "num_tweets", + "fieldName": "num_tweets" + } + ] + }, + { + "type": "arithmetic", + "name": "avg_tweet_len_half", + "fn": "/", + "fields": [ + { + "type": "fieldAccess", + "name": "avg_tweet_len", + "fieldName": "avg_tweet_len" + }, + { + "type": "constant", + "value": "2" + } + ] + }, + { + "type": "arithmetic", + "name": "avg_tweet_len_doubled", + "fn": "*", + "fields": [ + { + "type": "fieldAccess", + "name": "avg_tweet_len", + "fieldName": "avg_tweet_len" + }, + { + "type": "constant", + "value": "2" + } + ] + } + ], + "dimension": "user_name", + "metric": { + "type": "numeric", + "metric": "avg_tweet_len_doubled" + }, + "threshold": 2, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "user_name": "___soMALIa___", + "tweet_length": 539.0, + "avg_tweet_len_half": 269.5, + "avg_tweet_len_doubled": 1078.0, + "num_tweets": 1.0, + "avg_tweet_len": 539.0 + }, + { + "user_name": "SophiiiaSlr", + "tweet_length": 530.0, + "avg_tweet_len_half": 265.0, + "avg_tweet_len_doubled": 1060.0, + "num_tweets": 1.0, + "avg_tweet_len": 530.0 + } + ] + }, + { + "timestamp": "2013-01-02T00:00:00.000Z", + "result": [ + { + "user_name": "FallenReckless", + "tweet_length": 518.0, + "avg_tweet_len_half": 259.0, + "avg_tweet_len_doubled": 1036.0, + "num_tweets": 1.0, + "avg_tweet_len": 518.0 + }, + { + "user_name": "SigaMike", + "tweet_length": 514.0, + "avg_tweet_len_half": 257.0, + "avg_tweet_len_doubled": 1028.0, + "num_tweets": 1.0, + "avg_tweet_len": 514.0 + } + ] + }, + { + "timestamp": "2013-01-03T00:00:00.000Z", + "result": [ + { + "user_name": "Alejo_InReverse", + "tweet_length": 560.0, + "avg_tweet_len_half": 280.0, + "avg_tweet_len_doubled": 1120.0, + "num_tweets": 1.0, + "avg_tweet_len": 560.0 + }, + { + "user_name": "GavLeftHome", + "tweet_length": 506.0, + "avg_tweet_len_half": 253.0, + "avg_tweet_len_doubled": 1012.0, + "num_tweets": 1.0, + "avg_tweet_len": 506.0 + } + ] + } + ] + } +] diff --git a/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json b/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json new file mode 100644 index 00000000000..157e8b04750 --- /dev/null +++ b/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json @@ -0,0 +1,1063 @@ +[ + { + "description": "timeseries, 1 agg, all", + "query": { + "queryType": "timeseries", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + } + ], + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": { + "rows": 2390950 + } + } + ] + }, + { + "description": "timeseries, all aggs, all", + "query": { + "queryType": "timeseries", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + }, + { + "type": "doubleSum", + "fieldName": "variation", + "name": "variation" + }, + { + "type": "doubleSum", + "fieldName": "delta", + "name": "delta" + }, + { + "type": "approxHistogramFold", + "name": "delta_hist", + "fieldName": "delta_hist" + }, + { + "type": "hyperUnique", + "fieldName": "unique_users", + "name": "unique_users" + } + ], + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": { + "added": 9.11526338E8, + "count": 2815650, + "delta": 5.48967603E8, + "variation": 1.274085073E9, + "delta_hist": { + "breaks": [ + -2634692.25, -2048505.0, -1462317.75, -876130.4375, -289943.125, 296244.1875, + 882431.5, 1468619.0 + ], + "counts": [1.0, 2.0, 1.0, 56.0, 2815544.0, 41.0, 5.0] + }, + "unique_users": 229361.39005604674, + "deleted": -3.62558735E8, + "rows": 2390950 + } + } + ] + }, + { + "description": "timeseries, filtered, all aggs, all", + "query": { + "queryType": "timeseries", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "filter": { + "type": "and", + "fields": [ + { + "type": "selector", + "dimension": "namespace", + "value": "article" + }, + { + "type": "or", + "fields": [ + { + "type": "selector", + "dimension": "language", + "value": "en" + }, + { + "type": "selector", + "dimension": "unpatrolled", + "value": "0" + } + ] + } + ] + }, + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + }, + { + "type": "doubleSum", + "fieldName": "variation", + "name": "variation" + }, + { + "type": "doubleSum", + "fieldName": "delta", + "name": "delta" + }, + { + "type": "approxHistogramFold", + "name": "delta_hist", + "fieldName": "delta_hist" + }, + { + "type": "hyperUnique", + "fieldName": "unique_users", + "name": "unique_users" + } + ], + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": { + "added": 3.49393993E8, + "count": 1829240, + "delta": 2.24089868E8, + "variation": 4.74698118E8, + "delta_hist": { + "breaks": [ + -754245.3125, -565684.0, -377122.6875, -188561.359375, -0.03125, 188561.296875, + 377122.625, 565684.0 + ], + "counts": [1.0, 2.0, 11.0, 572.0, 1828642.0, 10.0, 2.0] + }, + "unique_users": 166138.2309016003, + "deleted": -1.25304125E8, + "rows": 1556534 + } + } + ] + }, + { + "description": "timeseries, 3 aggs, 1 post agg, all", + "query": { + "queryType": "timeseries", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfAddedDeletedConst", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "added", + "fieldName": "added" + }, + { + "type": "arithmetic", + "name": "", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "constant", + "name": "constant", + "value": 1000 + } + ] + } + ] + } + ], + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": { + "added": 9.11526338E8, + "count": 2815650, + "deleted": -3.62558735E8, + "sumOfAddedDeletedConst": 5.48968603E8, + "rows": 2390950 + } + } + ] + }, + { + "description": "topN, 1 agg", + "query": { + "queryType": "topN", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + } + ], + "dimension": "page", + "metric": "rows", + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "page": "Wikipedia:Vandalismusmeldung", + "rows": 991 + }, + { + "page": "Wikipedia:Administrators'_noticeboard/Incidents", + "rows": 990 + }, + { + "page": "Wikipedia:Administrator_intervention_against_vandalism", + "rows": 800 + } + ] + } + ] + }, + { + "description": "topN, all aggs, page dim, uniques metric", + "query": { + "queryType": "topN", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + }, + { + "type": "doubleSum", + "fieldName": "variation", + "name": "variation" + }, + { + "type": "doubleSum", + "fieldName": "delta", + "name": "delta" + }, + { + "type": "hyperUnique", + "fieldName": "unique_users", + "name": "unique_users" + } + ], + "dimension": "page", + "metric": "unique_users", + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "added": 1812960.0, + "count": 1697, + "page": "Wikipedia:Administrators'_noticeboard/Incidents", + "delta": 770071.0, + "variation": 2855849.0, + "unique_users": 323.21435881635085, + "deleted": -1042889.0, + "rows": 990 + }, + { + "added": 70162.0, + "count": 967, + "page": "2013", + "delta": 40872.0, + "variation": 99452.0, + "unique_users": 301.08619358578636, + "deleted": -29290.0, + "rows": 773 + }, + { + "added": 519152.0, + "count": 1700, + "page": "Wikipedia:Vandalismusmeldung", + "delta": -5446.0, + "variation": 1043750.0, + "unique_users": 298.7707608914404, + "deleted": -524598.0, + "rows": 991 + } + ] + } + ] + }, + { + "description": "topN, all aggs, page dim, count metric, filtered", + "query": { + "queryType": "topN", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "filter": { + "type": "and", + "fields": [ + { + "type": "selector", + "dimension": "namespace", + "value": "article" + }, + { + "type": "or", + "fields": [ + { + "type": "selector", + "dimension": "language", + "value": "en" + }, + { + "type": "selector", + "dimension": "unpatrolled", + "value": "0" + } + ] + } + ] + }, + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + }, + { + "type": "doubleSum", + "fieldName": "variation", + "name": "variation" + }, + { + "type": "doubleSum", + "fieldName": "delta", + "name": "delta" + }, + { + "type": "hyperUnique", + "fieldName": "unique_users", + "name": "unique_users" + } + ], + "dimension": "page", + "metric": "count", + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "added": 61739.0, + "count": 852, + "page": "2013", + "delta": 35313.0, + "variation": 88165.0, + "unique_users": 245.09832757908927, + "deleted": -26426.0, + "rows": 692 + }, + { + "added": 28288.0, + "count": 513, + "page": "Gérard_Depardieu", + "delta": 7027.0, + "variation": 49549.0, + "unique_users": 203.8133555888084, + "deleted": -21261.0, + "rows": 398 + }, + { + "added": 10951.0, + "count": 459, + "page": "Zichyújfalu", + "delta": 9030.0, + "variation": 12872.0, + "unique_users": 13.041435202975777, + "deleted": -1921.0, + "rows": 447 + } + ] + } + ] + }, + { + "description": "topN, all aggs, page dim, count metric, postAggs", + "query": { + "queryType": "topN", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "added" + }, + { + "type": "doubleSum", + "fieldName": "deleted", + "name": "deleted" + }, + { + "type": "doubleSum", + "fieldName": "variation", + "name": "variation" + }, + { + "type": "doubleSum", + "fieldName": "delta", + "name": "delta" + }, + { + "type": "hyperUnique", + "fieldName": "unique_users", + "name": "unique_users" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfAddedDeletedConst", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "added", + "fieldName": "added" + }, + { + "type": "arithmetic", + "name": "", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "constant", + "name": "constant", + "value": 1000 + } + ] + } + ] + } + ], + "dimension": "page", + "metric": "count", + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "added": 151409.0, + "count": 1770, + "page": "User:Cyde/List_of_candidates_for_speedy_deletion/Subpage", + "delta": 670.0, + "variation": 302148.0, + "unique_users": 1.0002442201269182, + "deleted": -150739.0, + "sumOfAddedDeletedConst": 1670.0, + "rows": 168 + }, + { + "added": 519152.0, + "count": 1700, + "page": "Wikipedia:Vandalismusmeldung", + "delta": -5446.0, + "variation": 1043750.0, + "unique_users": 298.7707608914404, + "deleted": -524598.0, + "sumOfAddedDeletedConst": -4446.0, + "rows": 991 + }, + { + "added": 1812960.0, + "count": 1697, + "page": "Wikipedia:Administrators'_noticeboard/Incidents", + "delta": 770071.0, + "variation": 2855849.0, + "unique_users": 323.21435881635085, + "deleted": -1042889.0, + "sumOfAddedDeletedConst": 771071.0, + "rows": 990 + } + ] + } + ] + }, + { + "description": "topN, lexicographic, two aggs, language dim, postAggs", + "query": { + "queryType": "topN", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfRowsAndCount", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "rows", + "fieldName": "rows" + }, + { + "type": "fieldAccess", + "name": "count", + "fieldName": "count" + } + ] + } + ], + "dimension": "language", + "metric": { + "type": "lexicographic", + "previousStop": "a" + }, + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "sumOfRowsAndCount": 63046.0, + "count": 33674, + "language": "ar", + "rows": 29372 + }, + { + "sumOfRowsAndCount": 26269.0, + "count": 13652, + "language": "bg", + "rows": 12617 + }, + { + "sumOfRowsAndCount": 60831.0, + "count": 32066, + "language": "ca", + "rows": 28765 + } + ] + } + ] + }, + { + "description": "topN, inverted, two aggs, namespace dim, postAggs", + "query": { + "queryType": "topN", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfRowsAndCount", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "rows", + "fieldName": "rows" + }, + { + "type": "fieldAccess", + "name": "count", + "fieldName": "count" + } + ] + } + ], + "dimension": "namespace", + "metric": { + "type": "inverted", + "metric": "count" + }, + "threshold": 3, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2013-01-01T00:00:00.000Z", + "result": [ + { + "sumOfRowsAndCount": 1553213.0, + "count": 835585, + "namespace": "wikipedia", + "rows": 717628 + }, + { + "sumOfRowsAndCount": 3653387.0, + "count": 1980065, + "namespace": "article", + "rows": 1673322 + } + ] + } + ] + }, + { + "description": "groupBy, two aggs, namespace dim, postAggs", + "query": { + "queryType": "groupBy", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfRowsAndCount", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "rows", + "fieldName": "rows" + }, + { + "type": "fieldAccess", + "name": "count", + "fieldName": "count" + } + ] + } + ], + "dimensions": ["namespace"], + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 3653387.0, + "count": 1980065, + "rows": 1673322, + "namespace": "article" + } + }, + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 1553213.0, + "count": 835585, + "rows": 717628, + "namespace": "wikipedia" + } + } + ] + }, + { + "description": "groupBy, two aggs, namespace + robot dim, postAggs", + "query": { + "queryType": "groupBy", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "longSum", + "fieldName": "count", + "name": "count" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "name": "sumOfRowsAndCount", + "fn": "+", + "fields": [ + { + "type": "fieldAccess", + "name": "rows", + "fieldName": "rows" + }, + { + "type": "fieldAccess", + "name": "count", + "fieldName": "count" + } + ] + } + ], + "dimensions": ["namespace", "robot"], + "limitSpec": { + "type": "default", + "limit": 3, + "orderBy": ["robot", "namespace"] + }, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 2268154.0, + "count": 1286354, + "robot": "0", + "rows": 981800, + "namespace": "article" + } + }, + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 1385233.0, + "count": 693711, + "robot": "1", + "rows": 691522, + "namespace": "article" + } + }, + { + "version": "v1", + "timestamp": "2013-01-01T00:00:00.000Z", + "event": { + "sumOfRowsAndCount": 878393.0, + "count": 492643, + "robot": "0", + "rows": 385750, + "namespace": "wikipedia" + } + } + ] + }, + { + "query": { + "queryType": "search", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "dataSource": "wikipedia_editstream", + "filter": { + "type": "and", + "fields": [ + { + "type": "selector", + "dimension": "namespace", + "value": "article" + }, + { + "type": "or", + "fields": [ + { + "type": "selector", + "dimension": "language", + "value": "en" + }, + { + "type": "selector", + "dimension": "unpatrolled", + "value": "0" + } + ] + } + ] + }, + "granularity": "all", + "searchDimensions": ["page", "namespace"], + "query": { + "type": "insensitive_contains", + "value": "league_of_legends" + }, + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2012-12-29T00:00:00.000Z", + "result": [ + { + "dimension": "page", + "value": "League_of_Legends" + }, + { + "dimension": "page", + "value": "The_best_ADs_in_The_League_of_legends" + } + ] + } + ] + }, + { + "query": { + "queryType": "timeBoundary", + "dataSource": "wikipedia_editstream", + "context": { + "useCache": "true", + "populateCache": "true", + "timeout": 60000 + } + }, + "expectedResults": [ + { + "timestamp": "2012-12-29T00:00:00.000Z", + "result": { + "minTime": "2012-12-29T00:00:00.000Z", + "maxTime": "2013-01-10T07:59:00.000Z" + } + } + ] + }, + { + "description": "segmentMetadata", + "query": { + "queryType": "segmentMetadata", + "dataSource": "wikipedia_editstream", + "intervals": ["2013-01-01T00:00:00.000/2013-01-08T00:00:00.000"], + "toInclude": { + "type": "list", + "columns": ["country_name", "language"] + } + }, + "expectedResults": [ + { + "id": "wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9", + "intervals": ["2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z"], + "columns": { + "country_name": { + "type": "STRING", + "size": 41922148, + "cardinality": 208, + "errorMessage": null + }, + "language": { + "type": "STRING", + "size": 8924222, + "cardinality": 36, + "errorMessage": null + } + }, + "size": 902457341 + } + ] + } +] diff --git a/integration-tests/src/test/resources/testng.xml b/integration-tests/src/test/resources/testng.xml new file mode 100644 index 00000000000..4e2b8f147f4 --- /dev/null +++ b/integration-tests/src/test/resources/testng.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/integration-tests/stop_cluster.sh b/integration-tests/stop_cluster.sh new file mode 100755 index 00000000000..b7bd21bac52 --- /dev/null +++ b/integration-tests/stop_cluster.sh @@ -0,0 +1,5 @@ +for node in druid-historical druid-coordinator druid-overlord druid-router druid-broker druid-middlemanager druid-zookeeper druid-metadata-storage; +do +docker stop $node +docker rm $node +done diff --git a/pom.xml b/pom.xml index 2126ba85b65..1c06f81a66d 100644 --- a/pom.xml +++ b/pom.xml @@ -55,6 +55,7 @@ processing server services + integration-tests extensions/cassandra-storage extensions/hdfs-storage @@ -509,6 +510,11 @@ + + org.testng + testng + 6.8.7 +