diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java index ab5f2bb8274..7332010da4d 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; @@ -29,6 +30,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -67,6 +69,8 @@ public class IntegrationTestImportTsv extends Configured implements Tool { private static final String NAME = IntegrationTestImportTsv.class.getSimpleName(); private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestImportTsv.class); + private static final String GENERATED_HFILE_FOLDER_PARAM_KEY = + "IntegrationTestImportTsv.generatedHFileFolder"; protected static final String simple_tsv = "row1\t1\tc1\tc2\n" + @@ -191,8 +195,8 @@ public class IntegrationTestImportTsv extends Configured implements Tool { void generateAndLoad(final TableName table) throws Exception { LOG.info("Running test testGenerateAndLoad."); String cf = "d"; - Path hfiles = new Path( - util.getDataTestDirOnTestFS(table.getNameAsString()), "hfiles"); + Path hfiles = initGeneratedHFilePath(table); + LOG.info("The folder where the HFiles will be generated: {}", hfiles.toString()); Map args = new HashMap<>(); args.put(ImportTsv.BULK_OUTPUT_CONF_KEY, hfiles.toString()); @@ -221,6 +225,12 @@ public class IntegrationTestImportTsv extends Configured implements Tool { System.err.println(format("%s [genericOptions]", NAME)); System.err.println(" Runs ImportTsv integration tests against a distributed cluster."); System.err.println(); + System.err.println(" Use '-D" + GENERATED_HFILE_FOLDER_PARAM_KEY + "=' to define a"); + System.err.println(" base folder for the generated HFiles. If HDFS Transparent Encryption"); + System.err.println(" is configured, then make sure to set this parameter to a folder in"); + System.err.println(" the same encryption zone in HDFS as the HBase root directory,"); + System.err.println(" otherwise the bulkload will fail."); + System.err.println(); ToolRunner.printGenericCommandUsage(System.err); return 1; } @@ -238,6 +248,27 @@ public class IntegrationTestImportTsv extends Configured implements Tool { return 0; } + private Path initGeneratedHFilePath(final TableName table) throws IOException { + String folderParam = getConf().getTrimmed(GENERATED_HFILE_FOLDER_PARAM_KEY); + if (folderParam == null || folderParam.isEmpty()) { + // by default, fall back to the test data dir + return new Path(util.getDataTestDirOnTestFS(table.getNameAsString()), "hfiles"); + } + + Path hfiles = new Path(folderParam, UUID.randomUUID().toString()); + FileSystem fs = util.getTestFileSystem(); + String shouldPreserve = System.getProperty("hbase.testing.preserve.testdir", "false"); + if (!Boolean.parseBoolean(shouldPreserve)) { + if (fs.getUri().getScheme().equals(FileSystem.getLocal(getConf()).getUri().getScheme())) { + File localFoler = new File(hfiles.toString()); + localFoler.deleteOnExit(); + } else { + fs.deleteOnExit(hfiles); + } + } + return hfiles; + } + public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); IntegrationTestingUtility.setUseDistributedCluster(conf);