HADOOP-17362. reduce RPC calls doing ls on HAR file (#2444). Contributed by Daryn Sharp and Ahmed Hussein

(cherry picked from commit 4bdba35b9f)
This commit is contained in:
Jim Brennan 2020-11-13 21:49:35 +00:00
parent cbc352ea2d
commit 118d8f5aa1
2 changed files with 37 additions and 38 deletions

View File

@ -35,6 +35,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
* This is an implementation of the Hadoop Archive
@ -511,41 +512,22 @@ public class HarFileSystem extends FileSystem {
if (!parentString.endsWith(Path.SEPARATOR)){
parentString += Path.SEPARATOR;
}
Path harPath = new Path(parentString);
int harlen = harPath.depth();
final Map<String, FileStatus> cache = new TreeMap<String, FileStatus>();
for (HarStatus hstatus : metadata.archive.values()) {
String child = hstatus.getName();
if ((child.startsWith(parentString))) {
Path thisPath = new Path(child);
if (thisPath.depth() == harlen + 1) {
statuses.add(toFileStatus(hstatus, cache));
}
}
for (String child: parent.children) {
Path p = new Path(parentString + child);
statuses.add(toFileStatus(metadata.archive.get(p)));
}
}
/**
* Combine the status stored in the index and the underlying status.
* @param h status stored in the index
* @param cache caching the underlying file statuses
* @return the combined file status
* @throws IOException
*/
private FileStatus toFileStatus(HarStatus h,
Map<String, FileStatus> cache) throws IOException {
FileStatus underlying = null;
if (cache != null) {
underlying = cache.get(h.partName);
}
if (underlying == null) {
final Path p = h.isDir? archivePath: new Path(archivePath, h.partName);
underlying = fs.getFileStatus(p);
if (cache != null) {
cache.put(h.partName, underlying);
}
}
private FileStatus toFileStatus(HarStatus h) throws IOException {
final Path p = h.isDir ? archivePath : new Path(archivePath, h.partName);
FileStatus underlying = metadata.getPartFileStatus(p);
long modTime = 0;
int version = metadata.getVersion();
@ -656,7 +638,7 @@ public class HarFileSystem extends FileSystem {
@Override
public FileStatus getFileStatus(Path f) throws IOException {
HarStatus hstatus = getFileHarStatus(f);
return toFileStatus(hstatus, null);
return toFileStatus(hstatus);
}
private HarStatus getFileHarStatus(Path f) throws IOException {
@ -813,7 +795,7 @@ public class HarFileSystem extends FileSystem {
if (hstatus.isDir()) {
fileStatusesInIndex(hstatus, statuses);
} else {
statuses.add(toFileStatus(hstatus, null));
statuses.add(toFileStatus(hstatus));
}
return statuses.toArray(new FileStatus[statuses.size()]);
@ -1126,7 +1108,8 @@ public class HarFileSystem extends FileSystem {
List<Store> stores = new ArrayList<Store>();
Map<Path, HarStatus> archive = new HashMap<Path, HarStatus>();
private Map<Path, FileStatus> partFileStatuses = new HashMap<Path, FileStatus>();
// keys are always the internal har path.
private Map<Path, FileStatus> partFileStatuses = new ConcurrentHashMap<>();
public HarMetaData(FileSystem fs, Path masterIndexPath, Path archiveIndexPath) {
this.fs = fs;
@ -1134,16 +1117,23 @@ public class HarFileSystem extends FileSystem {
this.archiveIndexPath = archiveIndexPath;
}
public FileStatus getPartFileStatus(Path partPath) throws IOException {
public FileStatus getPartFileStatus(Path path) throws IOException {
Path partPath = getPathInHar(path);
FileStatus status;
status = partFileStatuses.get(partPath);
if (status == null) {
status = fs.getFileStatus(partPath);
status = fs.getFileStatus(path);
partFileStatuses.put(partPath, status);
}
return status;
}
private void addPartFileStatuses(Path path) throws IOException {
for (FileStatus stat : fs.listStatus(path)) {
partFileStatuses.put(getPathInHar(stat.getPath()), stat);
}
}
public long getMasterIndexTimestamp() {
return masterIndexTimestamp;
}
@ -1200,16 +1190,22 @@ public class HarFileSystem extends FileSystem {
try {
FileStatus archiveStat = fs.getFileStatus(archiveIndexPath);
archiveIndexTimestamp = archiveStat.getModificationTime();
LineReader aLin;
// pre-populate part cache.
addPartFileStatuses(archiveIndexPath.getParent());
LineReader aLin = null;
// now start reading the real index file
long pos = -1;
for (Store s: stores) {
read = 0;
if (pos != s.begin) {
pos = s.begin;
aIn.seek(s.begin);
aLin = new LineReader(aIn, getConf());
while (read + s.begin < s.end) {
int tmp = aLin.readLine(line);
read += tmp;
}
while (pos < s.end) {
pos += aLin.readLine(line);
String lineFeed = line.toString();
String[] parsed = lineFeed.split(" ");
parsed[0] = decodeFileName(parsed[0]);

View File

@ -33,7 +33,10 @@ import java.net.URI;
import java.util.HashSet;
import java.util.Set;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
* This test class checks basic operations with {@link HarFileSystem} including