Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-3038 WALReaderFSDataInputStream.getPos() fails if Filesize > MAX_INT 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1002027 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2010-09-28 05:55:51 +00:00
parent 8032206236
commit d3a8c97ad8
2 changed files with 36 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -550,6 +550,8 @@ Release 0.21.0 - Unreleased
HBASE-3042 Use LO4J in SequenceFileLogReader HBASE-3042 Use LO4J in SequenceFileLogReader
(Nicolas Spiegelberg via Stack) (Nicolas Spiegelberg via Stack)
HBASE-2995 Incorrect dependency on Log class from Jetty HBASE-2995 Incorrect dependency on Log class from Jetty
HBASE-3038 WALReaderFSDataInputStream.getPos() fails if Filesize > MAX_INT
(Nicolas Spiegelberg via Stack)
IMPROVEMENTS IMPROVEMENTS
HBASE-1760 Cleanup TODOs in HTable HBASE-1760 Cleanup TODOs in HTable

43
src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
View File

@ -20,7 +20,7 @@
package org.apache.hadoop.hbase.regionserver.wal; package org.apache.hadoop.hbase.regionserver.wal;
import java.io.EOFException; import java.io.FilterInputStream;
import java.io.IOException; import java.io.IOException;
import java.lang.Class; import java.lang.Class;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
@ -78,18 +78,43 @@ public class SequenceFileLogReader implements HLog.Reader {
this.length = l; this.length = l;
} }
// This section can be confusing. It is specific to how HDFS works.
// Let me try to break it down. This is the problem:
//
// 1. HDFS DataNodes update the NameNode about a filename's length
// on block boundaries or when a file is closed. Therefore,
// if an RS dies, then the NN's fs.getLength() can be out of date
// 2. this.in.available() would work, but it returns int &
// therefore breaks for files > 2GB (happens on big clusters)
// 3. DFSInputStream.getFileLength() gets the actual length from the DNs
// 4. DFSInputStream is wrapped 2 levels deep : this.in.in
//
// So, here we adjust getPos() using getFileLength() so the
// SequenceFile.Reader constructor (aka: first invocation) comes out
// with the correct end of the file:
// this.end = in.getPos() + length;
@Override @Override
public long getPos() throws IOException { public long getPos() throws IOException {
if (this.firstGetPosInvocation) { if (this.firstGetPosInvocation) {
this.firstGetPosInvocation = false; this.firstGetPosInvocation = false;
// Tell a lie. We're doing this just so that this line up in long adjust = 0;
// SequenceFile.Reader constructor comes out with the correct length
// on the file: try {
// this.end = in.getPos() + length; Field fIn = FilterInputStream.class.getDeclaredField("in");
long available = this.in.available(); fIn.setAccessible(true);
// Length gets added up in the SF.Reader constructor so subtract the Object realIn = fIn.get(this.in);
// difference. If available < this.length, then return this.length. long realLength = ((Long)realIn.getClass().
return available >= this.length? available - this.length: this.length; getMethod("getFileLength", new Class<?> []{}).
invoke(realIn, new Object []{})).longValue();
assert(realLength >= this.length);
adjust = realLength - this.length;
} catch(Exception e) {
SequenceFileLogReader.LOG.warn(
"Error while trying to get accurate file length. " +
"Truncation / data loss may occur if RegionServers die.", e);
}
return adjust + super.getPos();
} }
return super.getPos(); return super.getPos();
} }