Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-17020. Improve RawFileSystem Performance (#2063) 

Contributed by : Mehakmeet Singh

Co-authored-by: Rajesh Balamohan
Co-authored-by: Mehakmeet Singh
This commit is contained in:
Mehakmeet Singh 2020-06-17 20:45:26 +05:30 committed by GitHub
parent 5b1a56f9f1
commit 2bfb22840a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
View File

@ -64,6 +64,7 @@
public class RawLocalFileSystem extends FileSystem { public class RawLocalFileSystem extends FileSystem {
static final URI NAME = URI.create("file:///"); static final URI NAME = URI.create("file:///");
private Path workingDir; private Path workingDir;
private long defaultBlockSize;
// Temporary workaround for HADOOP-9652. // Temporary workaround for HADOOP-9652.
private static boolean useDeprecatedFileStatus = true; private static boolean useDeprecatedFileStatus = true;
@ -100,6 +101,7 @@ public File pathToFile(Path path) {
public void initialize(URI uri, Configuration conf) throws IOException { public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf); super.initialize(uri, conf);
setConf(conf); setConf(conf);
defaultBlockSize = getDefaultBlockSize(new Path(uri));
} }
/******************************************************* /*******************************************************
@ -518,7 +520,12 @@ public FileStatus[] listStatus(Path f) throws IOException {
} }
return new FileStatus[] { return new FileStatus[] {
new DeprecatedRawLocalFileStatus(localf, new DeprecatedRawLocalFileStatus(localf,
getDefaultBlockSize(f), this) }; defaultBlockSize, this) };
}
@Override
public boolean exists(Path f) throws IOException {
return pathToFile(f).exists();
} }
protected boolean mkOneDir(File p2f) throws IOException { protected boolean mkOneDir(File p2f) throws IOException {
@ -663,7 +670,7 @@ private FileStatus deprecatedGetFileStatus(Path f) throws IOException {
File path = pathToFile(f); File path = pathToFile(f);
if (path.exists()) { if (path.exists()) {
return new DeprecatedRawLocalFileStatus(pathToFile(f), return new DeprecatedRawLocalFileStatus(pathToFile(f),
getDefaultBlockSize(f), this); defaultBlockSize, this);
} else { } else {
throw new FileNotFoundException("File " + f + " does not exist"); throw new FileNotFoundException("File " + f + " does not exist");
} }
@ -1051,7 +1058,7 @@ private FileStatus deprecatedGetFileLinkStatusInternal(final Path f)
private FileStatus getNativeFileLinkStatus(final Path f, private FileStatus getNativeFileLinkStatus(final Path f,
boolean dereference) throws IOException { boolean dereference) throws IOException {
checkPath(f); checkPath(f);
Stat stat = new Stat(f, getDefaultBlockSize(f), dereference, this); Stat stat = new Stat(f, defaultBlockSize, dereference, this);
FileStatus status = stat.getFileStatus(); FileStatus status = stat.getFileStatus();
return status; return status;
} }