Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-6991. Fix SequenceFile::Reader to honor file lengths and call

Browse Source 
openFile (cdouglas via omalley)


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1028473 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Owen O'Malley 2010-10-28 20:54:17 +00:00
parent 7afa9466f7
commit 0c462b223f
2 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -286,6 +286,9 @@ Trunk (unreleased changes)
HADOOP-6663. BlockDecompressorStream get EOF exception when decompressing HADOOP-6663. BlockDecompressorStream get EOF exception when decompressing
the file compressed from empty file. (Kang Xiao via tomwhite) the file compressed from empty file. (Kang Xiao via tomwhite)
HADOOP-6991. Fix SequenceFile::Reader to honor file lengths and call
openFile (cdouglas via omalley)
Release 0.21.1 - Unreleased Release 0.21.1 - Unreleased
IMPROVEMENTS IMPROVEMENTS

17
src/java/org/apache/hadoop/io/SequenceFile.java
View File

@ -951,7 +951,7 @@ public static Option compression(CompressionType value,
blockSizeOption.getValue(); blockSizeOption.getValue();
Progressable progress = progressOption == null ? null : Progressable progress = progressOption == null ? null :
progressOption.getValue(); progressOption.getValue();
out = fs.create(p, false, bufferSize, replication, blockSize, progress); out = fs.create(p, true, bufferSize, replication, blockSize, progress);
} else { } else {
out = streamOption.getValue(); out = streamOption.getValue();
} }
@ -1563,14 +1563,17 @@ public Reader(Configuration conf, Option... opts) throws IOException {
// figure out the real values // figure out the real values
Path filename = null; Path filename = null;
FSDataInputStream file; FSDataInputStream file;
long len = lenOpt == null ? Long.MAX_VALUE : lenOpt.getValue(); final long len;
if (fileOpt != null) { if (fileOpt != null) {
filename = fileOpt.getValue(); filename = fileOpt.getValue();
FileSystem fs = filename.getFileSystem(conf); FileSystem fs = filename.getFileSystem(conf);
int bufSize = bufOpt == null ? getBufferSize(conf): bufOpt.getValue(); int bufSize = bufOpt == null ? getBufferSize(conf): bufOpt.getValue();
file = fs.open(filename, bufSize); len = null == lenOpt
len = fs.getFileStatus(filename).getLen(); ? fs.getFileStatus(filename).getLen()
: lenOpt.getValue();
file = openFile(fs, filename, bufSize, len);
} else { } else {
len = null == lenOpt ? Long.MAX_VALUE : lenOpt.getValue();
file = streamOpt.getValue(); file = streamOpt.getValue();
} }
long start = startOpt == null ? 0 : startOpt.getValue(); long start = startOpt == null ? 0 : startOpt.getValue();
@ -1589,9 +1592,7 @@ public Reader(Configuration conf, Option... opts) throws IOException {
@Deprecated @Deprecated
public Reader(FileSystem fs, Path file, public Reader(FileSystem fs, Path file,
Configuration conf) throws IOException { Configuration conf) throws IOException {
initialize(file, this(conf, file(file.makeQualified(fs)));
fs.open(file, getBufferSize(conf)),
0L, fs.getFileStatus(file).getLen(), conf, false);
} }
/** /**
@ -1607,7 +1608,7 @@ public Reader(FileSystem fs, Path file,
@Deprecated @Deprecated
public Reader(FSDataInputStream in, int buffersize, public Reader(FSDataInputStream in, int buffersize,
long start, long length, Configuration conf) throws IOException { long start, long length, Configuration conf) throws IOException {
initialize(null, in, start, length, conf, false); this(conf, stream(in), start(start), length(length));
} }
/** Common work of the constructors. */ /** Common work of the constructors. */