Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20469 Directory used for sidelining old recovered edits files should be made configurable 

Signed-off-by: Andrew Purtell <apurtell@apache.org>
This commit is contained in:
Nihal Jain 2018-04-21 16:40:21 +05:30 committed by Andrew Purtell
parent 159435ee40
commit 092efb4274
2 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
View File

@ -460,6 +460,7 @@ public class WALSplitter {
* creating it if necessary. * creating it if necessary.
* @param logEntry * @param logEntry
* @param fileNameBeingSplit the file being split currently. Used to generate tmp file name. * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.
* @param tmpDirName of the directory used to sideline old recovered edits file
* @param conf * @param conf
* @return Path to file into which to dump split log edits. * @return Path to file into which to dump split log edits.
* @throws IOException * @throws IOException
@ -467,8 +468,7 @@ public class WALSplitter {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
@VisibleForTesting @VisibleForTesting
static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit, static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,
Configuration conf) String tmpDirName, Configuration conf) throws IOException {
throws IOException {
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
Path rootDir = FSUtils.getRootDir(conf); Path rootDir = FSUtils.getRootDir(conf);
Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName()); Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());
@ -483,7 +483,7 @@ public class WALSplitter {
return null; return null;
} }
if (fs.exists(dir) && fs.isFile(dir)) { if (fs.exists(dir) && fs.isFile(dir)) {
Path tmp = new Path("/tmp"); Path tmp = new Path(tmpDirName);
if (!fs.exists(tmp)) { if (!fs.exists(tmp)) {
fs.mkdirs(tmp); fs.mkdirs(tmp);
} }
@ -1512,8 +1512,10 @@ public class WALSplitter {
* @return a path with a write for that path. caller should close. * @return a path with a write for that path. caller should close.
*/ */
WriterAndPath createWAP(byte[] region, Entry entry) throws IOException { WriterAndPath createWAP(byte[] region, Entry entry) throws IOException {
String tmpDirName = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY,
HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY);
Path regionedits = getRegionSplitEditsPath(entry, Path regionedits = getRegionSplitEditsPath(entry,
fileBeingSplit.getPath().getName(), conf); fileBeingSplit.getPath().getName(), tmpDirName, conf);
if (regionedits == null) { if (regionedits == null) {
return null; return null;
} }

7
hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
View File

@ -130,6 +130,7 @@ public class TestWALSplit {
private Path OLDLOGDIR; private Path OLDLOGDIR;
private Path CORRUPTDIR; private Path CORRUPTDIR;
private Path TABLEDIR; private Path TABLEDIR;
private String TMPDIRNAME;
private static final int NUM_WRITERS = 10; private static final int NUM_WRITERS = 10;
private static final int ENTRIES = 10; // entries per writer per region private static final int ENTRIES = 10; // entries per writer per region
@ -191,6 +192,8 @@ public class TestWALSplit {
OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME); OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME);
CORRUPTDIR = new Path(HBASELOGDIR, HConstants.CORRUPT_DIR_NAME); CORRUPTDIR = new Path(HBASELOGDIR, HConstants.CORRUPT_DIR_NAME);
TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME); TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME);
TMPDIRNAME = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY,
HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY);
REGIONS.clear(); REGIONS.clear();
Collections.addAll(REGIONS, "bbb", "ccc"); Collections.addAll(REGIONS, "bbb", "ccc");
InstrumentedLogWriter.activateFailure = false; InstrumentedLogWriter.activateFailure = false;
@ -391,7 +394,7 @@ public class TestWALSplit {
TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),
new WALEdit()); new WALEdit());
Path p = WALSplitter.getRegionSplitEditsPath(entry, Path p = WALSplitter.getRegionSplitEditsPath(entry,
FILENAME_BEING_SPLIT, conf); FILENAME_BEING_SPLIT, TMPDIRNAME, conf);
String parentOfParent = p.getParent().getParent().getName(); String parentOfParent = p.getParent().getParent().getName();
assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
} }
@ -417,7 +420,7 @@ public class TestWALSplit {
fs.createNewFile(parent); // create a recovered.edits file fs.createNewFile(parent); // create a recovered.edits file
Path p = WALSplitter.getRegionSplitEditsPath(entry, Path p = WALSplitter.getRegionSplitEditsPath(entry,
FILENAME_BEING_SPLIT, conf); FILENAME_BEING_SPLIT, TMPDIRNAME, conf);
String parentOfParent = p.getParent().getParent().getName(); String parentOfParent = p.getParent().getParent().getName();
assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
WALFactory.createRecoveredEditsWriter(fs, p, conf).close(); WALFactory.createRecoveredEditsWriter(fs, p, conf).close();