Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

svn merge -c 1442755 from trunk for HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in HarFileSystem. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1442756 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2013-02-05 21:24:05 +00:00
parent 7126958734
commit 2d5f195730
4 changed files with 81 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -286,6 +286,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9260. Hadoop version may be not correct when starting name node or
data node. (Chris Nauroth via jlowe)
HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in
HarFileSystem. (Chris Nauroth via szetszwo)
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES

35
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
View File

@ -30,8 +30,11 @@ import java.util.TreeMap;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.util.Progressable;
@ -50,6 +53,9 @@ import org.apache.hadoop.util.Progressable;
*/
public class HarFileSystem extends FilterFileSystem {
private static final Log LOG = LogFactory.getLog(HarFileSystem.class);
public static final int VERSION = 3;
private static final Map<URI, HarMetaData> harMetaCache =
@ -1025,12 +1031,18 @@ public class HarFileSystem extends FilterFileSystem {
}
private void parseMetaData() throws IOException {
FSDataInputStream in = fs.open(masterIndexPath);
Text line;
long read;
FSDataInputStream in = null;
LineReader lin = null;
try {
in = fs.open(masterIndexPath);
FileStatus masterStat = fs.getFileStatus(masterIndexPath);
masterIndexTimestamp = masterStat.getModificationTime();
LineReader lin = new LineReader(in, getConf());
Text line = new Text();
long read = lin.readLine(line);
lin = new LineReader(in, getConf());
line = new Text();
read = lin.readLine(line);
// the first line contains the version of the index file
String versionLine = line.toString();
@ -1055,14 +1067,12 @@ public class HarFileSystem extends FilterFileSystem {
endHash));
line.clear();
}
try {
// close the master index
lin.close();
} catch(IOException io){
// do nothing just a read.
} finally {
IOUtils.cleanup(LOG, lin, in);
}
FSDataInputStream aIn = fs.open(archiveIndexPath);
try {
FileStatus archiveStat = fs.getFileStatus(archiveIndexPath);
archiveIndexTimestamp = archiveStat.getModificationTime();
LineReader aLin;
@ -1082,11 +1092,8 @@ public class HarFileSystem extends FilterFileSystem {
line.clear();
}
}
try {
// close the archive index
aIn.close();
} catch(IOException io) {
// do nothing just a read.
} finally {
IOUtils.cleanup(LOG, aIn);
}
}
}

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.util;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
@ -39,7 +40,7 @@ import org.apache.hadoop.io.Text;
*/
@InterfaceAudience.LimitedPrivate({"MapReduce"})
@InterfaceStability.Unstable
public class LineReader {
public class LineReader implements Closeable {
private static final int DEFAULT_BUFFER_SIZE = 64 * 1024;
private int bufferSize = DEFAULT_BUFFER_SIZE;
private InputStream in;

15
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
View File

@ -28,6 +28,7 @@ import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Shell;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -46,8 +47,18 @@ public class TestHarFileSystemBasics {
private static final String ROOT_PATH = System.getProperty("test.build.data",
"build/test/data");
private static final Path rootPath = new Path(
new File(ROOT_PATH).getAbsolutePath() + "/localfs");
private static final Path rootPath;
static {
String root = new Path(new File(ROOT_PATH).getAbsolutePath(), "localfs")
.toUri().getPath();
// Strip drive specifier on Windows, which would make the HAR URI invalid and
// cause tests to fail.
if (Shell.WINDOWS) {
root = root.substring(root.indexOf(':') + 1);
}
rootPath = new Path(root);
}
// NB: .har suffix is necessary
private static final Path harPath = new Path(rootPath, "path1/path2/my.har");