From c7d864d3bccf82601c757f44c859eaeae34cfc1d Mon Sep 17 00:00:00 2001 From: abhagraw <99210446+abhagraw@users.noreply.github.com> Date: Thu, 16 Mar 2023 11:04:43 +0530 Subject: [PATCH] Update container creation in AzureTestUtil.java (#13911) * 1. Handling deletion/creation of container created during the previously run test in AzureTestUtil.java. 2. Adding/updating log messages and comments in Azure and GCS deep storage tests. --- .github/workflows/reusable-revised-its.yml | 2 +- .github/workflows/revised-its.yml | 2 +- ...ractAzureInputSourceParallelIndexTest.java | 1 + .../indexer/ITGcsToGcsParallelIndexTest.java | 14 ++++++------- .../msq/ITAzureSQLBasedIngestionTest.java | 4 ++-- .../msq/ITGcsSQLBasedIngestionTest.java | 9 ++++++++ .../druid/testsEx/utils/AzureTestUtil.java | 21 ++++++++++++++++++- 7 files changed, 40 insertions(+), 13 deletions(-) diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index 91ff249b2c9..5a220d5755a 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -48,7 +48,7 @@ env: SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 jobs: - test: # Github job that runs a given revised/new IT against retrieved cached druid docker image + test: # GitHub job that runs a given revised/new IT against retrieved cached druid docker image name: ${{ inputs.it }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}) runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index e4cf409ee6b..83cad23f42f 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Github workflow that runs revised/new ITs +# GitHub workflow that runs revised/new ITs on: workflow_call: workflow_dispatch: diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractAzureInputSourceParallelIndexTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractAzureInputSourceParallelIndexTest.java index 3098956a8cc..81852df11c7 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractAzureInputSourceParallelIndexTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractAzureInputSourceParallelIndexTest.java @@ -42,6 +42,7 @@ public class AbstractAzureInputSourceParallelIndexTest extends AbstractCloudInpu public static void uploadDataFilesToAzure() { try { + LOG.info("Uploading files to Azure"); azure = new AzureTestUtil(); // Creating a container with name set in AZURE_CONTAINER env variable. azure.createStorageContainer(); diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITGcsToGcsParallelIndexTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITGcsToGcsParallelIndexTest.java index 0eaef4e64c9..ca202797bd0 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITGcsToGcsParallelIndexTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITGcsToGcsParallelIndexTest.java @@ -31,15 +31,13 @@ import java.util.List; /** * IMPORTANT: - * To run this test, you must: - * 1) Set the bucket and path for your data. This can be done by setting -Ddruid.test.config.cloudBucket and - * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path" in the config file. - * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and wikipedia_index_data3.json - * located in integration-tests/src/test/resources/data/batch_index/json to your GCS at the location set in step 1. - * 3) Provide -Doverride.config.path= with gcs configs set. See - * integration-tests/docker/environment-configs/override-examples/gcs for env vars to provide. - * 4) Provide -Dresource.file.dir.path= with folder that contains GOOGLE_APPLICATION_CREDENTIALS file + * To run this test, you must set the following env variables in the build environment - + * GOOGLE_PREFIX - path inside the bucket where the test data files will be uploaded + * GOOGLE_BUCKET - Google cloud bucket name + * GOOGLE_APPLICATION_CREDENTIALS - path to the json file containing google cloud credentials + * Google Cloud Storage setup in druid */ + @RunWith(DruidTestRunner.class) @Category(GcsDeepStorage.class) public class ITGcsToGcsParallelIndexTest extends AbstractGcsInputSourceParallelIndexTest diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureSQLBasedIngestionTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureSQLBasedIngestionTest.java index c54e8916967..54f207704a8 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureSQLBasedIngestionTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureSQLBasedIngestionTest.java @@ -22,7 +22,7 @@ package org.apache.druid.testsEx.msq; import junitparams.Parameters; import junitparams.naming.TestCaseName; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.testsEx.categories.S3DeepStorage; +import org.apache.druid.testsEx.categories.AzureDeepStorage; import org.apache.druid.testsEx.config.DruidTestRunner; import org.apache.druid.testsEx.indexer.AbstractAzureInputSourceParallelIndexTest; import org.junit.Test; @@ -41,7 +41,7 @@ import java.util.List; */ @RunWith(DruidTestRunner.class) -@Category(S3DeepStorage.class) +@Category(AzureDeepStorage.class) public class ITAzureSQLBasedIngestionTest extends AbstractAzureInputSourceParallelIndexTest { private static final String CLOUD_INGEST_SQL = "/multi-stage-query/wikipedia_cloud_index_msq.sql"; diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITGcsSQLBasedIngestionTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITGcsSQLBasedIngestionTest.java index a6031c6e1fa..125b30ef81f 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITGcsSQLBasedIngestionTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITGcsSQLBasedIngestionTest.java @@ -31,6 +31,15 @@ import org.junit.runner.RunWith; import java.util.List; +/** + * IMPORTANT: + * To run this test, you must set the following env variables in the build environment - + * GOOGLE_PREFIX - path inside the bucket where the test data files will be uploaded + * GOOGLE_BUCKET - Google cloud bucket name + * GOOGLE_APPLICATION_CREDENTIALS - path to the json file containing google cloud credentials + * Google Cloud Storage setup in druid + */ + @RunWith(DruidTestRunner.class) @Category(GcsDeepStorage.class) public class ITGcsSQLBasedIngestionTest extends AbstractGcsInputSourceParallelIndexTest diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/AzureTestUtil.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/AzureTestUtil.java index 4b31206a1d5..90be0cbd464 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/AzureTestUtil.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/AzureTestUtil.java @@ -25,6 +25,7 @@ import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.testing.utils.ITRetryUtil; import java.io.File; import java.io.IOException; @@ -83,9 +84,26 @@ public class AzureTestUtil public void createStorageContainer() throws URISyntaxException, StorageException { + LOG.info("Creating azure container " + AZURE_CONTAINER); CloudBlobContainer container = azureStorageClient.getContainerReference(AZURE_CONTAINER); // Create the container if it does not exist. - container.createIfNotExists(); + + // From the azure documentation - + // When a container is deleted, a container with the same name can't be created for at least 30 seconds. + // The container might not be available for more than 30 seconds if the service is still processing the request. + // While the container is being deleted, attempts to create a container of the same name fail with status + // code 409 (Conflict). The service indicates that the container is being deleted. + // All other operations, including operations on any blobs under the container, + // fail with status code 404 (Not Found) while the container is being deleted. + ITRetryUtil.retryUntil( + () -> container.createIfNotExists(), + true, + 10000, + 13, + "Create Azure container : " + AZURE_CONTAINER + " " + ); + + LOG.info("Azure container " + AZURE_CONTAINER + " created"); } public void deleteStorageContainer() throws URISyntaxException, StorageException @@ -109,6 +127,7 @@ public class AzureTestUtil // Create or overwrite the "myimage.jpg" blob with contents from a local file. File source = new File(filePath); CloudBlockBlob blob = container.getBlockBlobReference(DRUID_CLOUD_PATH + '/' + source.getName()); + LOG.info("Uploading file " + DRUID_CLOUD_PATH + '/' + source.getName() + " in azure container " + AZURE_CONTAINER); blob.upload(Files.newInputStream(source.toPath()), source.length()); } }