From 73a6baaeb641889b86afc55934e78502825b042b Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn <52679095+maytasm@users.noreply.github.com> Date: Tue, 7 Apr 2020 08:37:37 -1000 Subject: [PATCH] change hadoop inputSource IT to use parallel batch ingestion (#9616) --- ...a => AbstractHdfsInputSourceParallelIndexTest.java} | 10 ++++++++-- ...exTest.java => ITHdfsToAzureParallelIndexTest.java} | 2 +- ...ndexTest.java => ITHdfsToGcsParallelIndexTest.java} | 2 +- ...dexTest.java => ITHdfsToHdfsParallelIndexTest.java} | 2 +- ...IndexTest.java => ITHdfsToS3ParallelIndexTest.java} | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) rename integration-tests/src/test/java/org/apache/druid/tests/indexer/{AbstractHdfsInputSourceSimpleIndexTest.java => AbstractHdfsInputSourceParallelIndexTest.java} (89%) rename integration-tests/src/test/java/org/apache/druid/tests/indexer/{ITHdfsToAzureSimpleIndexTest.java => ITHdfsToAzureParallelIndexTest.java} (95%) rename integration-tests/src/test/java/org/apache/druid/tests/indexer/{ITHdfsToGcsSimpleIndexTest.java => ITHdfsToGcsParallelIndexTest.java} (95%) rename integration-tests/src/test/java/org/apache/druid/tests/indexer/{ITHdfsToHdfsSimpleIndexTest.java => ITHdfsToHdfsParallelIndexTest.java} (94%) rename integration-tests/src/test/java/org/apache/druid/tests/indexer/{ITHdfsToS3SimpleIndexTest.java => ITHdfsToS3ParallelIndexTest.java} (95%) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractHdfsInputSourceSimpleIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractHdfsInputSourceParallelIndexTest.java similarity index 89% rename from integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractHdfsInputSourceSimpleIndexTest.java rename to integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractHdfsInputSourceParallelIndexTest.java index 393823de74c..874eb78b8a7 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractHdfsInputSourceSimpleIndexTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractHdfsInputSourceParallelIndexTest.java @@ -20,6 +20,7 @@ package org.apache.druid.tests.indexer; import com.google.common.collect.ImmutableList; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; import org.testng.annotations.DataProvider; @@ -29,9 +30,9 @@ import java.util.List; import java.util.UUID; import java.util.function.Function; -public abstract class AbstractHdfsInputSourceSimpleIndexTest extends AbstractITBatchIndexTest +public abstract class AbstractHdfsInputSourceParallelIndexTest extends AbstractITBatchIndexTest { - private static final String INDEX_TASK = "/indexer/wikipedia_cloud_simple_index_task.json"; + private static final String INDEX_TASK = "/indexer/wikipedia_cloud_index_task.json"; private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_index_queries.json"; private static final String INDEX_DATASOURCE = "wikipedia_index_test_" + UUID.randomUUID(); private static final String INPUT_SOURCE_PATHS_KEY = "paths"; @@ -70,6 +71,11 @@ public abstract class AbstractHdfsInputSourceSimpleIndexTest extends AbstractITB "%%INPUT_SOURCE_TYPE%%", "hdfs" ); + spec = StringUtils.replace( + spec, + "%%PARTITIONS_SPEC%%", + jsonMapper.writeValueAsString(new DynamicPartitionsSpec(null, null)) + ); spec = StringUtils.replace( spec, "%%INPUT_SOURCE_PROPERTY_KEY%%", diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToAzureSimpleIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToAzureParallelIndexTest.java similarity index 95% rename from integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToAzureSimpleIndexTest.java rename to integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToAzureParallelIndexTest.java index 42a2a0ccdbb..9caf7440549 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToAzureSimpleIndexTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToAzureParallelIndexTest.java @@ -37,7 +37,7 @@ import java.util.List; */ @Test(groups = TestNGGroup.AZURE_DEEP_STORAGE) @Guice(moduleFactory = DruidTestModuleFactory.class) -public class ITHdfsToAzureSimpleIndexTest extends AbstractHdfsInputSourceSimpleIndexTest +public class ITHdfsToAzureParallelIndexTest extends AbstractHdfsInputSourceParallelIndexTest { @Test(dataProvider = "resources") public void testHdfsIndexData(Pair hdfsInputSource) throws Exception diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToGcsSimpleIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToGcsParallelIndexTest.java similarity index 95% rename from integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToGcsSimpleIndexTest.java rename to integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToGcsParallelIndexTest.java index b37f184c3ab..a2e1fb71416 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToGcsSimpleIndexTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToGcsParallelIndexTest.java @@ -38,7 +38,7 @@ import java.util.List; */ @Test(groups = TestNGGroup.GCS_DEEP_STORAGE) @Guice(moduleFactory = DruidTestModuleFactory.class) -public class ITHdfsToGcsSimpleIndexTest extends AbstractHdfsInputSourceSimpleIndexTest +public class ITHdfsToGcsParallelIndexTest extends AbstractHdfsInputSourceParallelIndexTest { @Test(dataProvider = "resources") public void testHdfsIndexData(Pair hdfsInputSource) throws Exception diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToHdfsSimpleIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToHdfsParallelIndexTest.java similarity index 94% rename from integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToHdfsSimpleIndexTest.java rename to integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToHdfsParallelIndexTest.java index 05f052c1a7f..c09121e3f86 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToHdfsSimpleIndexTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToHdfsParallelIndexTest.java @@ -36,7 +36,7 @@ import java.util.List; */ @Test(groups = TestNGGroup.HDFS_DEEP_STORAGE) @Guice(moduleFactory = DruidTestModuleFactory.class) -public class ITHdfsToHdfsSimpleIndexTest extends AbstractHdfsInputSourceSimpleIndexTest +public class ITHdfsToHdfsParallelIndexTest extends AbstractHdfsInputSourceParallelIndexTest { @Test(dataProvider = "resources") public void testHdfsIndexData(Pair hdfsInputSource) throws Exception diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToS3SimpleIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToS3ParallelIndexTest.java similarity index 95% rename from integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToS3SimpleIndexTest.java rename to integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToS3ParallelIndexTest.java index f92c675d4f9..69825412c1d 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToS3SimpleIndexTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHdfsToS3ParallelIndexTest.java @@ -37,7 +37,7 @@ import java.util.List; */ @Test(groups = TestNGGroup.S3_DEEP_STORAGE) @Guice(moduleFactory = DruidTestModuleFactory.class) -public class ITHdfsToS3SimpleIndexTest extends AbstractHdfsInputSourceSimpleIndexTest +public class ITHdfsToS3ParallelIndexTest extends AbstractHdfsInputSourceParallelIndexTest { @Test(dataProvider = "resources") public void testHdfsIndexData(Pair hdfsInputSource) throws Exception