Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25363 Improve performance of HFileLinkCleaner by using ReadWriteLock instead of synchronize

This commit is contained in:
haxiaolin 2020-12-08 17:21:16 +08:00 committed by niuyulin
parent 5016219d3c
commit 56dd3eba81
1 changed files with 56 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.master.cleaner; package org.apache.hadoop.hbase.master.cleaner;
import java.io.IOException; import java.io.IOException;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -44,13 +45,20 @@ public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
private static final Logger LOG = LoggerFactory.getLogger(HFileLinkCleaner.class); private static final Logger LOG = LoggerFactory.getLogger(HFileLinkCleaner.class);
private FileSystem fs = null; private FileSystem fs = null;
private ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@Override @Override
public synchronized boolean isFileDeletable(FileStatus fStat) { public boolean isFileDeletable(FileStatus fStat) {
if (this.fs == null) return false; lock.readLock().lock();
try {
if (this.fs == null) {
return false;
}
Path filePath = fStat.getPath(); Path filePath = fStat.getPath();
// HFile Link is always deletable // HFile Link is always deletable
if (HFileLink.isHFileLink(filePath)) return true; if (HFileLink.isHFileLink(filePath)) {
return true;
}
// If the file is inside a link references directory, means that it is a back ref link. // If the file is inside a link references directory, means that it is a back ref link.
// The back ref can be deleted only if the referenced file doesn't exists. // The back ref can be deleted only if the referenced file doesn't exists.
@ -60,8 +68,8 @@ public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
try { try {
// Also check if the HFile is in the HBASE_TEMP_DIRECTORY; this is where the referenced // Also check if the HFile is in the HBASE_TEMP_DIRECTORY; this is where the referenced
// file gets created when cloning a snapshot. // file gets created when cloning a snapshot.
hfilePath = HFileLink.getHFileFromBackReference( hfilePath = HFileLink.getHFileFromBackReference(new Path(
new Path(CommonFSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY), filePath); CommonFSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY), filePath);
if (fs.exists(hfilePath)) { if (fs.exists(hfilePath)) {
return false; return false;
} }
@ -70,13 +78,13 @@ public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
if (fs.exists(hfilePath)) { if (fs.exists(hfilePath)) {
return false; return false;
} }
hfilePath = hfilePath = HFileLink.getHFileFromBackReference(CommonFSUtils.getRootDir(getConf()),
HFileLink.getHFileFromBackReference(CommonFSUtils.getRootDir(getConf()), filePath); filePath);
return !fs.exists(hfilePath); return !fs.exists(hfilePath);
} catch (IOException e) { } catch (IOException e) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Couldn't verify if the referenced file still exists, keep it just in case: " + LOG.debug("Couldn't verify if the referenced file still exists, keep it just in case: "
hfilePath); + hfilePath);
} }
return false; return false;
} }
@ -89,18 +97,23 @@ public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
return CommonFSUtils.listStatus(fs, backRefDir) == null; return CommonFSUtils.listStatus(fs, backRefDir) == null;
} catch (IOException e) { } catch (IOException e) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Couldn't get the references, not deleting file, just in case. filePath=" LOG.debug(
+ filePath + ", backRefDir=" + backRefDir); "Couldn't get the references, not deleting file, just in case. filePath=" +
filePath + ", backRefDir=" + backRefDir);
} }
return false; return false;
} }
} finally {
lock.readLock().unlock();
}
} }
@Override @Override
public synchronized void setConf(Configuration conf) { public void setConf(Configuration conf) {
super.setConf(conf); super.setConf(conf);
// setup filesystem // setup filesystem
lock.writeLock().lock();
try { try {
this.fs = FileSystem.get(this.getConf()); this.fs = FileSystem.get(this.getConf());
} catch (IOException e) { } catch (IOException e) {
@ -109,6 +122,8 @@ public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
+ FileSystem.FS_DEFAULT_NAME_KEY + "=" + FileSystem.FS_DEFAULT_NAME_KEY + "="
+ getConf().get(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS)); + getConf().get(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS));
} }
} finally {
lock.writeLock().unlock();
} }
} }
} }