Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18002 Investigate why bucket cache filling up in file mode in an 

exisiting file  is slower (Ram)
This commit is contained in:
Ramkrishna 2017-07-06 11:20:00 +05:30
parent 1049025e1d
commit 381a151d14
3 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
View File

@ -227,7 +227,7 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[] bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath, int ioErrorsTolerationDuration)
throws FileNotFoundException, IOException {
this.ioEngine = getIOEngineFromName(ioEngineName, capacity);
this.ioEngine = getIOEngineFromName(ioEngineName, capacity, persistencePath);
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
if (blockNumCapacity >= Integer.MAX_VALUE) {
@ -309,10 +309,11 @@ public class BucketCache implements BlockCache, HeapSize {
* Get the IOEngine from the IO engine name
* @param ioEngineName
* @param capacity
* @param persistencePath
* @return the IOEngine
* @throws IOException
*/
private IOEngine getIOEngineFromName(String ioEngineName, long capacity)
private IOEngine getIOEngineFromName(String ioEngineName, long capacity, String persistencePath)
throws IOException {
if (ioEngineName.startsWith("file:") || ioEngineName.startsWith("files:")) {
// In order to make the usage simple, we only need the prefix 'files:' in
@ -320,7 +321,7 @@ public class BucketCache implements BlockCache, HeapSize {
// the compatibility
String[] filePaths = ioEngineName.substring(ioEngineName.indexOf(":") + 1)
.split(FileIOEngine.FILE_DELIMITER);
return new FileIOEngine(capacity, filePaths);
return new FileIOEngine(capacity, persistencePath != null, filePaths);
} else if (ioEngineName.startsWith("offheap")) {
return new ByteBufferIOEngine(capacity, true);
} else if (ioEngineName.startsWith("heap")) {

15
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
View File

@ -52,11 +52,24 @@ public class FileIOEngine implements IOEngine {
private FileReadAccessor readAccessor = new FileReadAccessor();
private FileWriteAccessor writeAccessor = new FileWriteAccessor();
public FileIOEngine(long capacity, String... filePaths) throws IOException {
public FileIOEngine(long capacity, boolean maintainPersistence, String... filePaths)
throws IOException {
this.sizePerFile = capacity / filePaths.length;
this.capacity = this.sizePerFile * filePaths.length;
this.filePaths = filePaths;
this.fileChannels = new FileChannel[filePaths.length];
if (!maintainPersistence) {
for (String filePath : filePaths) {
File file = new File(filePath);
if (file.exists()) {
if (LOG.isDebugEnabled()) {
LOG.debug("File " + filePath + " already exists. Deleting!!");
}
file.delete();
// If deletion fails still we can manage with the writes
}
}
}
this.rafs = new RandomAccessFile[filePaths.length];
for (int i = 0; i < filePaths.length; i++) {
String filePath = filePaths[i];

2
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
View File

@ -58,7 +58,7 @@ public class TestFileIOEngine {
boundaryStopPositions.add(sizePerFile * i + 1);
}
boundaryStopPositions.add(sizePerFile * filePaths.length - 1);
FileIOEngine fileIOEngine = new FileIOEngine(totalCapacity, filePaths);
FileIOEngine fileIOEngine = new FileIOEngine(totalCapacity, false, filePaths);
try {
for (int i = 0; i < 500; i++) {
int len = (int) Math.floor(Math.random() * 100);