From 6991735f73bc9bfed81fe2757699a418da111d90 Mon Sep 17 00:00:00 2001 From: Surekha Date: Fri, 8 Mar 2019 16:40:59 -0800 Subject: [PATCH] Fix and add sys IT tests to travis script (#7208) * Add sys IT tests to travis script * minor fixes * Modify the test queries * modify query --- ci/travis_script_integration_part2.sh | 2 +- integration-tests/README.md | 5 +++++ .../druid/tests/indexer/AbstractITBatchIndexTest.java | 11 +++++++++-- .../indexer/ITSystemTableBatchIndexTaskTest.java | 4 ++-- .../druid/tests/query/ITSystemTableQueryTest.java | 3 +-- .../indexer/sys_segment_batch_index_queries.json | 4 ++-- .../test/resources/queries/sys_segment_queries.json | 6 +----- 7 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ci/travis_script_integration_part2.sh b/ci/travis_script_integration_part2.sh index 24cd7970ece..8c9f5eebd76 100755 --- a/ci/travis_script_integration_part2.sh +++ b/ci/travis_script_integration_part2.sh @@ -21,6 +21,6 @@ set -e pushd $TRAVIS_BUILD_DIR/integration-tests -mvn verify -P integration-tests -Dit.test=ITUnionQueryTest,ITNestedQueryPushDownTest,ITTwitterQueryTest,ITWikipediaQueryTest,ITBasicAuthConfigurationTest,ITTLSTest +mvn verify -P integration-tests -Dit.test=ITUnionQueryTest,ITNestedQueryPushDownTest,ITTwitterQueryTest,ITWikipediaQueryTest,ITBasicAuthConfigurationTest,ITTLSTest,ITSystemTableQueryTest,ITSystemTableBatchIndexTaskTest popd diff --git a/integration-tests/README.md b/integration-tests/README.md index eeafc2f0495..55ec1855b1f 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -215,3 +215,8 @@ This will tell the test framework that the test class needs to be constructed us 2) FromFileTestQueryHelper - reads queries with expected results from file and executes them and verifies the results using ResultVerifier Refer ITIndexerTest as an example on how to use dependency Injection + +### Register new tests for Travis CI + +Once you add new integration tests, don't forget to add them to `{DRUID_ROOT}/ci/travis_script_integration.sh` +or `{DRUID_ROOT}/ci/travis_script_integration_part2.sh` for Travis CI to run them. diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractITBatchIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractITBatchIndexTest.java index 121fd7a9bda..df95de927c1 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractITBatchIndexTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractITBatchIndexTest.java @@ -144,9 +144,16 @@ public class AbstractITBatchIndexTest extends AbstractIndexerTest String dataSource, String indexTaskFilePath, String queryFilePath - ) + ) throws IOException { - submitTaskAndWait(indexTaskFilePath, dataSource, false); + final String fullDatasourceName = dataSource + config.getExtraDatasourceNameSuffix(); + final String taskSpec = StringUtils.replace( + getTaskAsString(indexTaskFilePath), + "%%DATASOURCE%%", + fullDatasourceName + ); + + submitTaskAndWait(taskSpec, fullDatasourceName, false); try { sqlQueryHelper.testQueriesFromFile(queryFilePath, 2); } diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITSystemTableBatchIndexTaskTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITSystemTableBatchIndexTaskTest.java index d06155d31f2..c138c9fe75d 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITSystemTableBatchIndexTaskTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITSystemTableBatchIndexTaskTest.java @@ -30,7 +30,7 @@ import java.io.Closeable; public class ITSystemTableBatchIndexTaskTest extends AbstractITBatchIndexTest { - private static final Logger LOG = new Logger(ITCompactionTaskTest.class); + private static final Logger LOG = new Logger(ITSystemTableBatchIndexTaskTest.class); private static String INDEX_TASK = "/indexer/wikipedia_index_task.json"; private static String SYSTEM_QUERIES_RESOURCE = "/indexer/sys_segment_batch_index_queries.json"; private static String INDEX_DATASOURCE = "wikipedia_index_test"; @@ -40,7 +40,7 @@ public class ITSystemTableBatchIndexTaskTest extends AbstractITBatchIndexTest { LOG.info("Starting batch index sys table queries"); try ( - final Closeable indexCloseable = unloader(INDEX_DATASOURCE) + final Closeable indexCloseable = unloader(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); ) { doIndexTestSqlTest( INDEX_DATASOURCE, diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java index 378fc5a39b1..82089f4b1ef 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java @@ -19,7 +19,6 @@ package org.apache.druid.tests.query; -import com.google.common.base.Throwables; import com.google.inject.Inject; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.clients.CoordinatorResourceTestClient; @@ -65,7 +64,7 @@ public class ITSystemTableQueryTest this.queryHelper.testQueriesFromFile(SYSTEM_QUERIES_RESOURCE, 2); } catch (Exception e) { - throw Throwables.propagate(e); + throw new RuntimeException(e); } } } diff --git a/integration-tests/src/test/resources/indexer/sys_segment_batch_index_queries.json b/integration-tests/src/test/resources/indexer/sys_segment_batch_index_queries.json index f49b387eda2..b8e863b95f4 100644 --- a/integration-tests/src/test/resources/indexer/sys_segment_batch_index_queries.json +++ b/integration-tests/src/test/resources/indexer/sys_segment_batch_index_queries.json @@ -1,7 +1,7 @@ [ { "query": { - "query": "SELECT count(*) FROM sys.segments WHERE datasource='wikipedia_index_test'" + "query": "SELECT count(*) FROM sys.segments WHERE datasource LIKE 'wikipedia_index_test%'" }, "expectedResults": [ { @@ -21,7 +21,7 @@ }, { "query": { - "query": "SELECT status FROM sys.tasks" + "query": "SELECT status AS status FROM sys.tasks WHERE datasource LIKE 'wikipedia_index_test%' GROUP BY 1" }, "expectedResults": [ { diff --git a/integration-tests/src/test/resources/queries/sys_segment_queries.json b/integration-tests/src/test/resources/queries/sys_segment_queries.json index d414af71fe1..8e074107135 100644 --- a/integration-tests/src/test/resources/queries/sys_segment_queries.json +++ b/integration-tests/src/test/resources/queries/sys_segment_queries.json @@ -1,17 +1,13 @@ [ { "query": { - "query": "SELECT datasource, count(*) FROM sys.segments GROUP BY 1" + "query": "SELECT datasource, count(*) FROM sys.segments WHERE datasource='wikipedia_editstream' OR datasource='twitterstream' GROUP BY 1 " }, "expectedResults": [ { "datasource": "wikipedia_editstream", "EXPR$1": 1 }, - { - "datasource": "wikipedia", - "EXPR$1": 1 - }, { "datasource": "twitterstream", "EXPR$1": 3