From 20347e0c86375693c7a23489559ff199299b7a45 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 25 Jan 2022 10:14:26 -0800 Subject: [PATCH] Wait for datasource to be ready for SQL in integration tests (#12189) * Wait for datasource to be ready for SQL in integration tests * add limit to the check query --- .../druid/testing/utils/DataLoaderHelper.java | 49 +++++++++++++++++++ .../testing/utils/SqlTestQueryHelper.java | 26 ++++++++++ .../tests/query/ITBroadcastJoinQueryTest.java | 8 +-- .../druid/tests/query/ITJdbcQueryTest.java | 9 ++-- .../druid/tests/query/ITQueryErrorTest.java | 9 ++-- .../druid/tests/query/ITSqlCancelTest.java | 9 ++-- .../tests/query/ITSystemTableQueryTest.java | 13 ++--- 7 files changed, 93 insertions(+), 30 deletions(-) create mode 100644 integration-tests/src/main/java/org/apache/druid/testing/utils/DataLoaderHelper.java diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/DataLoaderHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/DataLoaderHelper.java new file mode 100644 index 00000000000..76b4300067e --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/DataLoaderHelper.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.utils; + +import com.google.inject.Inject; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.testing.clients.CoordinatorResourceTestClient; + +public final class DataLoaderHelper +{ + @Inject + private SqlTestQueryHelper sqlTestQueryHelper; + + @Inject + private CoordinatorResourceTestClient coordinator; + + public void waitUntilDatasourceIsReady(String datasource) + { + ITRetryUtil.retryUntilTrue( + () -> coordinator.areSegmentsLoaded(datasource), + StringUtils.format( + "Segment Load for datasource [%s]", + datasource + ) + ); + + ITRetryUtil.retryUntilTrue( + () -> sqlTestQueryHelper.isDatasourceLoadedInSQL(datasource), + StringUtils.format("Waiting for [%s] to be ready for SQL queries", datasource) + ); + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java index 373f3cbee0f..fe9a3243c5f 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java @@ -22,11 +22,14 @@ package org.apache.druid.testing.utils; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.sql.http.SqlQuery; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.clients.SqlResourceTestClient; public class SqlTestQueryHelper extends AbstractTestQueryHelper { + private static final Logger LOG = new Logger(SqlTestQueryHelper.class); @Inject public SqlTestQueryHelper( @@ -43,4 +46,27 @@ public class SqlTestQueryHelper extends AbstractTestQueryHelper coordinatorClient.areSegmentsLoaded(BROADCAST_JOIN_DATASOURCE), "broadcast segment load" - ); + dataLoaderHelper.waitUntilDatasourceIsReady(BROADCAST_JOIN_DATASOURCE); // query metadata until druid schema is refreshed and datasource is available joinable ITRetryUtil.retryUntilTrue( diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java index 46fb17706fb..6e3944029e1 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java @@ -26,9 +26,8 @@ import org.apache.druid.https.SSLClientConfig; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.testing.IntegrationTestingConfig; -import org.apache.druid.testing.clients.CoordinatorResourceTestClient; import org.apache.druid.testing.guice.DruidTestModuleFactory; -import org.apache.druid.testing.utils.ITRetryUtil; +import org.apache.druid.testing.utils.DataLoaderHelper; import org.apache.druid.tests.TestNGGroup; import org.testng.Assert; import org.testng.annotations.BeforeMethod; @@ -77,7 +76,7 @@ public class ITJdbcQueryTest SSLClientConfig sslConfig; @Inject - private CoordinatorResourceTestClient coordinatorClient; + private DataLoaderHelper dataLoaderHelper; @BeforeMethod public void before() @@ -108,9 +107,7 @@ public class ITJdbcQueryTest ) }; // ensure that wikipedia segments are loaded completely - ITRetryUtil.retryUntilTrue( - () -> coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE), "wikipedia segment load" - ); + dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE); } @Test diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITQueryErrorTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITQueryErrorTest.java index 9abc941c2e0..fd8ce5aabc7 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITQueryErrorTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITQueryErrorTest.java @@ -24,9 +24,8 @@ import com.google.inject.Inject; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.QueryContexts; import org.apache.druid.server.coordination.ServerManagerForQueryErrorTest; -import org.apache.druid.testing.clients.CoordinatorResourceTestClient; import org.apache.druid.testing.guice.DruidTestModuleFactory; -import org.apache.druid.testing.utils.ITRetryUtil; +import org.apache.druid.testing.utils.DataLoaderHelper; import org.apache.druid.testing.utils.SqlTestQueryHelper; import org.apache.druid.testing.utils.TestQueryHelper; import org.apache.druid.tests.TestNGGroup; @@ -69,7 +68,7 @@ public class ITQueryErrorTest private static final String SQL_PLAN_FAILURE_RESOURCE = "/queries/sql_plan_failure_query.json"; @Inject - private CoordinatorResourceTestClient coordinatorClient; + private DataLoaderHelper dataLoaderHelper; @Inject private TestQueryHelper queryHelper; @Inject @@ -81,9 +80,7 @@ public class ITQueryErrorTest public void before() { // ensure that wikipedia segments are loaded completely - ITRetryUtil.retryUntilTrue( - () -> coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE), "wikipedia segment load" - ); + dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE); } @Test(expectedExceptions = {RuntimeException.class}, expectedExceptionsMessageRegExp = "(?s).*400.*") diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java index e8a3fe5c875..3d3d43e75b9 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java @@ -30,10 +30,9 @@ import org.apache.druid.query.QueryException; import org.apache.druid.query.QueryInterruptedException; import org.apache.druid.sql.http.SqlQuery; import org.apache.druid.testing.IntegrationTestingConfig; -import org.apache.druid.testing.clients.CoordinatorResourceTestClient; import org.apache.druid.testing.clients.SqlResourceTestClient; import org.apache.druid.testing.guice.DruidTestModuleFactory; -import org.apache.druid.testing.utils.ITRetryUtil; +import org.apache.druid.testing.utils.DataLoaderHelper; import org.apache.druid.testing.utils.SqlTestQueryHelper; import org.apache.druid.tests.TestNGGroup; import org.jboss.netty.handler.codec.http.HttpResponseStatus; @@ -61,7 +60,7 @@ public class ITSqlCancelTest private static final int NUM_QUERIES = 3; @Inject - private CoordinatorResourceTestClient coordinatorClient; + private DataLoaderHelper dataLoaderHelper; @Inject private SqlTestQueryHelper sqlHelper; @Inject @@ -75,9 +74,7 @@ public class ITSqlCancelTest public void before() { // ensure that wikipedia segments are loaded completely - ITRetryUtil.retryUntilTrue( - () -> coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE), "wikipedia segment load" - ); + dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE); } @Test diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java index bb638ec7e33..df00488c6af 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java @@ -21,9 +21,8 @@ package org.apache.druid.tests.query; import com.google.inject.Inject; import org.apache.druid.testing.IntegrationTestingConfig; -import org.apache.druid.testing.clients.CoordinatorResourceTestClient; import org.apache.druid.testing.guice.DruidTestModuleFactory; -import org.apache.druid.testing.utils.ITRetryUtil; +import org.apache.druid.testing.utils.DataLoaderHelper; import org.apache.druid.testing.utils.SqlTestQueryHelper; import org.apache.druid.tests.TestNGGroup; import org.testng.annotations.BeforeMethod; @@ -39,7 +38,7 @@ public class ITSystemTableQueryTest private static final String SYSTEM_QUERIES_RESOURCE = "/queries/sys_queries.json"; @Inject - CoordinatorResourceTestClient coordinatorClient; + DataLoaderHelper dataLoaderHelper; @Inject private SqlTestQueryHelper queryHelper; @Inject @@ -49,14 +48,10 @@ public class ITSystemTableQueryTest public void before() { // ensure that wikipedia segments are loaded completely - ITRetryUtil.retryUntilTrue( - () -> coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE), "wikipedia segment load" - ); + dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE); // ensure that the twitter segments are loaded completely - ITRetryUtil.retryUntilTrue( - () -> coordinatorClient.areSegmentsLoaded(TWITTER_DATA_SOURCE), "twitter segment load" - ); + dataLoaderHelper.waitUntilDatasourceIsReady(TWITTER_DATA_SOURCE); } @Test