From 50bae96e8b44913faeb7c64a79430ffd7cb843a4 Mon Sep 17 00:00:00 2001 From: George Shiqi Wu Date: Thu, 1 Feb 2024 09:18:49 -0500 Subject: [PATCH] Add azure integrationt ests (#15799) --- ...ractCloudInputSourceParallelIndexTest.java | 6 ++ .../indexer/ITAzureV2ParallelIndexTest.java | 52 +++++++++++++++++ .../msq/ITAzureV2SQLBasedIngestionTest.java | 57 +++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITAzureV2ParallelIndexTest.java create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureV2SQLBasedIngestionTest.java diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractCloudInputSourceParallelIndexTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractCloudInputSourceParallelIndexTest.java index 51d8a20e2c3..a920d928b76 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractCloudInputSourceParallelIndexTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractCloudInputSourceParallelIndexTest.java @@ -48,9 +48,11 @@ public abstract class AbstractCloudInputSourceParallelIndexTest extends Abstract private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json"; private static final String GOOGLE = "google"; private static final String AZURE = "azure"; + private static final String AZURE_V2 = "azureStorage"; private static final String GOOGLE_PREFIX = "googlePrefix"; private static final String GOOGLE_BUCKET = "googleBucket"; private static final String AZURE_CONTAINER = "azureContainer"; + private static final String AZURE_STORAGE_ACCOUNT = "azureAccount"; private static final Logger LOG = new Logger(AbstractCloudInputSourceParallelIndexTest.class); String indexDatasource = "wikipedia_cloud_index_test_"; @@ -106,6 +108,8 @@ public abstract class AbstractCloudInputSourceParallelIndexTest extends Abstract { if (GOOGLE.equals(inputSourceType)) { return config.getProperty(GOOGLE_PREFIX); + } else if (AZURE_V2.equals(inputSourceType)) { + return config.getProperty(AZURE_CONTAINER) + "/" + config.getCloudPath(); } else { return config.getCloudPath(); } @@ -117,6 +121,8 @@ public abstract class AbstractCloudInputSourceParallelIndexTest extends Abstract return config.getProperty(GOOGLE_BUCKET); } else if (AZURE.equals(inputSourceType)) { return config.getProperty(AZURE_CONTAINER); + } else if (AZURE_V2.equals(inputSourceType)) { + return config.getProperty(AZURE_STORAGE_ACCOUNT); } else { return config.getCloudBucket(); } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITAzureV2ParallelIndexTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITAzureV2ParallelIndexTest.java new file mode 100644 index 00000000000..df539eb5903 --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITAzureV2ParallelIndexTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.indexer; + +import junitparams.Parameters; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.testsEx.categories.AzureDeepStorage; +import org.apache.druid.testsEx.config.DruidTestRunner; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.List; + +/** + * IMPORTANT: + * To run this test, you must set the following env variables in the build environment - + * DRUID_CLOUD_PATH - path inside the container where the test data files will be uploaded + *

+ * The AZURE account, key and container should be set in AZURE_ACCOUNT, AZURE_KEY and AZURE_CONTAINER respectively. + *

+ * Azure Deep Storage setup in druid + */ + +@RunWith(DruidTestRunner.class) +@Category(AzureDeepStorage.class) +public class ITAzureV2ParallelIndexTest extends AbstractAzureInputSourceParallelIndexTest +{ + @Test + @Parameters(method = "resources") + public void testAzureIndexData(Pair> azureInputSource) throws Exception + { + doTest(azureInputSource, new Pair<>(false, false), "azureStorage"); + } +} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureV2SQLBasedIngestionTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureV2SQLBasedIngestionTest.java new file mode 100644 index 00000000000..31486f5f4ed --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureV2SQLBasedIngestionTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.msq; + +import junitparams.Parameters; +import junitparams.naming.TestCaseName; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.testsEx.categories.AzureDeepStorage; +import org.apache.druid.testsEx.config.DruidTestRunner; +import org.apache.druid.testsEx.indexer.AbstractAzureInputSourceParallelIndexTest; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.List; + +/** + * IMPORTANT: + * To run this test, you must set the following env variables in the build environment - + *

+ * The AZURE account, key and container should be set in AZURE_ACCOUNT, AZURE_KEY and AZURE_CONTAINER respectively. + *

+ * Azure Deep Storage setup in druid + */ + +@RunWith(DruidTestRunner.class) +@Category(AzureDeepStorage.class) +public class ITAzureV2SQLBasedIngestionTest extends AbstractAzureInputSourceParallelIndexTest +{ + private static final String CLOUD_INGEST_SQL = "/multi-stage-query/wikipedia_cloud_index_msq.sql"; + private static final String INDEX_QUERIES_FILE = "/multi-stage-query/wikipedia_index_queries.json"; + + @Test + @Parameters(method = "resources") + @TestCaseName("Test_{index} ({0})") + public void testSQLBasedBatchIngestion(Pair> azureStorageInputSource) + { + doMSQTest(azureStorageInputSource, CLOUD_INGEST_SQL, INDEX_QUERIES_FILE, "azureStorage"); + } +}