From a3e7d36f2e1ce0a48511a4d30727728b6ec90449 Mon Sep 17 00:00:00 2001 From: Yutong Xiao Date: Tue, 18 Jan 2022 15:08:32 +0800 Subject: [PATCH] HBASE-26678 Backport HBASE-26579 Set storage policy of recovered edits when wal storage type is configured (#4037) Signed-off-by: Reid Chan --- .../apache/hadoop/hbase/wal/WALSplitter.java | 5 +++++ .../apache/hadoop/hbase/wal/TestWALSplit.java | 21 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index d6c18131554..c3434f9f6cb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -569,6 +569,7 @@ public class WALSplitter { * logEntry: e.g. /hbase/some_table/2323432434/recovered.edits/2332. * This method also ensures existence of RECOVERED_EDITS_DIR under the region * creating it if necessary. + * And also set storage policy for RECOVERED_EDITS_DIR if WAL_STORAGE_POLICY is configured. * @param logEntry * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name. * @param tmpDirName of the directory used to sideline old recovered edits file @@ -601,6 +602,10 @@ public class WALSplitter { if (!walFS.exists(dir) && !walFS.mkdirs(dir)) { LOG.warn("mkdir failed on " + dir); + } else { + String storagePolicy = + conf.get(HConstants.WAL_STORAGE_POLICY, HConstants.DEFAULT_WAL_STORAGE_POLICY); + FSUtils.setStoragePolicy(walFS, dir, storagePolicy); } // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now. // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index be80d78817c..2a1c32219d7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -1177,6 +1177,17 @@ public class TestWALSplit { } } + @Test + public void testRecoveredEditsStoragePolicy() throws IOException { + conf.set(HConstants.WAL_STORAGE_POLICY, "ALL_SSD"); + try { + Path path = createRecoveredEditsPathForRegion(); + assertEquals("ALL_SSD", fs.getStoragePolicy(path.getParent()).getName()); + } finally { + conf.unset(HConstants.WAL_STORAGE_POLICY); + } + } + private Writer generateWALs(int leaveOpen) throws IOException { return generateWALs(NUM_WRITERS, ENTRIES, leaveOpen, 0); } @@ -1425,4 +1436,14 @@ public class TestWALSplit { in2.close(); return true; } + + private Path createRecoveredEditsPathForRegion() throws IOException { + byte[] encoded = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + long now = EnvironmentEdgeManager.currentTime(); + Entry entry = new Entry( + new WALKey(encoded, TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), + new WALEdit()); + return WALSplitter + .getRegionSplitEditsPath(entry, FILENAME_BEING_SPLIT, TMPDIRNAME, conf); + } }