From 092efb42749bf7fc6ad338c96aae8e7b9d3a2c74 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sat, 21 Apr 2018 16:40:21 +0530 Subject: [PATCH] HBASE-20469 Directory used for sidelining old recovered edits files should be made configurable Signed-off-by: Andrew Purtell --- .../java/org/apache/hadoop/hbase/wal/WALSplitter.java | 10 ++++++---- .../java/org/apache/hadoop/hbase/wal/TestWALSplit.java | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 65d5fb7ee25..ac327c5b0a2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -460,6 +460,7 @@ public class WALSplitter { * creating it if necessary. * @param logEntry * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name. + * @param tmpDirName of the directory used to sideline old recovered edits file * @param conf * @return Path to file into which to dump split log edits. * @throws IOException @@ -467,8 +468,7 @@ public class WALSplitter { @SuppressWarnings("deprecation") @VisibleForTesting static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit, - Configuration conf) - throws IOException { + String tmpDirName, Configuration conf) throws IOException { FileSystem fs = FileSystem.get(conf); Path rootDir = FSUtils.getRootDir(conf); Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName()); @@ -483,7 +483,7 @@ public class WALSplitter { return null; } if (fs.exists(dir) && fs.isFile(dir)) { - Path tmp = new Path("/tmp"); + Path tmp = new Path(tmpDirName); if (!fs.exists(tmp)) { fs.mkdirs(tmp); } @@ -1512,8 +1512,10 @@ public class WALSplitter { * @return a path with a write for that path. caller should close. */ WriterAndPath createWAP(byte[] region, Entry entry) throws IOException { + String tmpDirName = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, + HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY); Path regionedits = getRegionSplitEditsPath(entry, - fileBeingSplit.getPath().getName(), conf); + fileBeingSplit.getPath().getName(), tmpDirName, conf); if (regionedits == null) { return null; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index 0d5aa0d2ff5..7a5abfd98c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -130,6 +130,7 @@ public class TestWALSplit { private Path OLDLOGDIR; private Path CORRUPTDIR; private Path TABLEDIR; + private String TMPDIRNAME; private static final int NUM_WRITERS = 10; private static final int ENTRIES = 10; // entries per writer per region @@ -191,6 +192,8 @@ public class TestWALSplit { OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME); CORRUPTDIR = new Path(HBASELOGDIR, HConstants.CORRUPT_DIR_NAME); TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME); + TMPDIRNAME = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, + HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY); REGIONS.clear(); Collections.addAll(REGIONS, "bbb", "ccc"); InstrumentedLogWriter.activateFailure = false; @@ -391,7 +394,7 @@ public class TestWALSplit { TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), new WALEdit()); Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, conf); + FILENAME_BEING_SPLIT, TMPDIRNAME, conf); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); } @@ -417,7 +420,7 @@ public class TestWALSplit { fs.createNewFile(parent); // create a recovered.edits file Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, conf); + FILENAME_BEING_SPLIT, TMPDIRNAME, conf); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); WALFactory.createRecoveredEditsWriter(fs, p, conf).close();