HADOOP-17362. reduce RPC calls doing ls on HAR file (#2444). Contributed by Daryn Sharp and Ahmed Hussein
(cherry picked from commit 4bdba35b9f
)
This commit is contained in:
parent
cbc352ea2d
commit
118d8f5aa1
|
@ -35,6 +35,7 @@ import java.net.URI;
|
|||
import java.net.URISyntaxException;
|
||||
import java.net.URLDecoder;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* This is an implementation of the Hadoop Archive
|
||||
|
@ -511,41 +512,22 @@ public class HarFileSystem extends FileSystem {
|
|||
if (!parentString.endsWith(Path.SEPARATOR)){
|
||||
parentString += Path.SEPARATOR;
|
||||
}
|
||||
Path harPath = new Path(parentString);
|
||||
int harlen = harPath.depth();
|
||||
final Map<String, FileStatus> cache = new TreeMap<String, FileStatus>();
|
||||
|
||||
for (HarStatus hstatus : metadata.archive.values()) {
|
||||
String child = hstatus.getName();
|
||||
if ((child.startsWith(parentString))) {
|
||||
Path thisPath = new Path(child);
|
||||
if (thisPath.depth() == harlen + 1) {
|
||||
statuses.add(toFileStatus(hstatus, cache));
|
||||
}
|
||||
}
|
||||
for (String child: parent.children) {
|
||||
Path p = new Path(parentString + child);
|
||||
statuses.add(toFileStatus(metadata.archive.get(p)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine the status stored in the index and the underlying status.
|
||||
* @param h status stored in the index
|
||||
* @param cache caching the underlying file statuses
|
||||
* @return the combined file status
|
||||
* @throws IOException
|
||||
*/
|
||||
private FileStatus toFileStatus(HarStatus h,
|
||||
Map<String, FileStatus> cache) throws IOException {
|
||||
FileStatus underlying = null;
|
||||
if (cache != null) {
|
||||
underlying = cache.get(h.partName);
|
||||
}
|
||||
if (underlying == null) {
|
||||
final Path p = h.isDir? archivePath: new Path(archivePath, h.partName);
|
||||
underlying = fs.getFileStatus(p);
|
||||
if (cache != null) {
|
||||
cache.put(h.partName, underlying);
|
||||
}
|
||||
}
|
||||
private FileStatus toFileStatus(HarStatus h) throws IOException {
|
||||
final Path p = h.isDir ? archivePath : new Path(archivePath, h.partName);
|
||||
FileStatus underlying = metadata.getPartFileStatus(p);
|
||||
|
||||
long modTime = 0;
|
||||
int version = metadata.getVersion();
|
||||
|
@ -656,7 +638,7 @@ public class HarFileSystem extends FileSystem {
|
|||
@Override
|
||||
public FileStatus getFileStatus(Path f) throws IOException {
|
||||
HarStatus hstatus = getFileHarStatus(f);
|
||||
return toFileStatus(hstatus, null);
|
||||
return toFileStatus(hstatus);
|
||||
}
|
||||
|
||||
private HarStatus getFileHarStatus(Path f) throws IOException {
|
||||
|
@ -813,7 +795,7 @@ public class HarFileSystem extends FileSystem {
|
|||
if (hstatus.isDir()) {
|
||||
fileStatusesInIndex(hstatus, statuses);
|
||||
} else {
|
||||
statuses.add(toFileStatus(hstatus, null));
|
||||
statuses.add(toFileStatus(hstatus));
|
||||
}
|
||||
|
||||
return statuses.toArray(new FileStatus[statuses.size()]);
|
||||
|
@ -1126,7 +1108,8 @@ public class HarFileSystem extends FileSystem {
|
|||
|
||||
List<Store> stores = new ArrayList<Store>();
|
||||
Map<Path, HarStatus> archive = new HashMap<Path, HarStatus>();
|
||||
private Map<Path, FileStatus> partFileStatuses = new HashMap<Path, FileStatus>();
|
||||
// keys are always the internal har path.
|
||||
private Map<Path, FileStatus> partFileStatuses = new ConcurrentHashMap<>();
|
||||
|
||||
public HarMetaData(FileSystem fs, Path masterIndexPath, Path archiveIndexPath) {
|
||||
this.fs = fs;
|
||||
|
@ -1134,16 +1117,23 @@ public class HarFileSystem extends FileSystem {
|
|||
this.archiveIndexPath = archiveIndexPath;
|
||||
}
|
||||
|
||||
public FileStatus getPartFileStatus(Path partPath) throws IOException {
|
||||
public FileStatus getPartFileStatus(Path path) throws IOException {
|
||||
Path partPath = getPathInHar(path);
|
||||
FileStatus status;
|
||||
status = partFileStatuses.get(partPath);
|
||||
if (status == null) {
|
||||
status = fs.getFileStatus(partPath);
|
||||
status = fs.getFileStatus(path);
|
||||
partFileStatuses.put(partPath, status);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
private void addPartFileStatuses(Path path) throws IOException {
|
||||
for (FileStatus stat : fs.listStatus(path)) {
|
||||
partFileStatuses.put(getPathInHar(stat.getPath()), stat);
|
||||
}
|
||||
}
|
||||
|
||||
public long getMasterIndexTimestamp() {
|
||||
return masterIndexTimestamp;
|
||||
}
|
||||
|
@ -1200,16 +1190,22 @@ public class HarFileSystem extends FileSystem {
|
|||
try {
|
||||
FileStatus archiveStat = fs.getFileStatus(archiveIndexPath);
|
||||
archiveIndexTimestamp = archiveStat.getModificationTime();
|
||||
LineReader aLin;
|
||||
|
||||
// pre-populate part cache.
|
||||
addPartFileStatuses(archiveIndexPath.getParent());
|
||||
LineReader aLin = null;
|
||||
|
||||
// now start reading the real index file
|
||||
long pos = -1;
|
||||
for (Store s: stores) {
|
||||
read = 0;
|
||||
if (pos != s.begin) {
|
||||
pos = s.begin;
|
||||
aIn.seek(s.begin);
|
||||
aLin = new LineReader(aIn, getConf());
|
||||
while (read + s.begin < s.end) {
|
||||
int tmp = aLin.readLine(line);
|
||||
read += tmp;
|
||||
}
|
||||
|
||||
while (pos < s.end) {
|
||||
pos += aLin.readLine(line);
|
||||
String lineFeed = line.toString();
|
||||
String[] parsed = lineFeed.split(" ");
|
||||
parsed[0] = decodeFileName(parsed[0]);
|
||||
|
|
|
@ -33,7 +33,10 @@ import java.net.URI;
|
|||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
|
||||
/**
|
||||
* This test class checks basic operations with {@link HarFileSystem} including
|
||||
|
|
Loading…
Reference in New Issue