Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13227. Add a method to calculate cumulative diff over multiple snapshots in DirectoryDiffList. Contributed by Shashikant Banerjee

This commit is contained in:
Tsz-Wo Nicholas Sze 2018-03-06 13:35:03 -08:00
parent 871d0d39fa
commit 346caa2095
2 changed files with 38 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
View File

@ -235,6 +235,12 @@ abstract class AbstractINodeDiffList<N extends INode,
return diff == null ? Snapshot.CURRENT_STATE_ID : diff.getSnapshotId();
}
public final int getDiffIndexById(final int snapshotId) {
int diffIndex = diffs.binarySearch(snapshotId);
diffIndex = diffIndex < 0 ? (-diffIndex - 1) : diffIndex;
return diffIndex;
}
final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) {
if (diffs == null) {
return null;
@ -247,10 +253,10 @@ abstract class AbstractINodeDiffList<N extends INode,
}
final int size = diffs.size();
int earlierDiffIndex = diffs.binarySearch(earlier.getId());
int earlierDiffIndex = getDiffIndexById(earlier.getId());
int laterDiffIndex = later == null ? size
: diffs.binarySearch(later.getId());
if (-earlierDiffIndex - 1 == size) {
: getDiffIndexById(later.getId());
if (earlierDiffIndex == size) {
// if the earlierSnapshot is after the latest SnapshotDiff stored in
// diffs, no modification happened after the earlierSnapshot
return null;
@ -260,10 +266,6 @@ abstract class AbstractINodeDiffList<N extends INode,
// before it, no modification happened before the laterSnapshot
return null;
}
earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1)
: earlierDiffIndex;
laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1)
: laterDiffIndex;
return new int[]{earlierDiffIndex, laterDiffIndex};
}
@ -304,7 +306,7 @@ abstract class AbstractINodeDiffList<N extends INode,
}
return diff;
}
@Override
public Iterator<D> iterator() {
return diffs != null ? diffs.iterator() : Collections.emptyIterator();

36
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
View File

@ -228,12 +228,18 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
private List<INode> initChildren() {
if (children == null) {
final ChildrenDiff combined = new ChildrenDiff();
for (DirectoryDiff d = DirectoryDiff.this; d != null;
d = d.getPosterior()) {
DirectoryDiffList directoryDiffList =
currentDir.getDirectoryWithSnapshotFeature().diffs;
final int diffIndex =
directoryDiffList.getDiffIndexById(getSnapshotId());
List<DirectoryDiff> diffList = directoryDiffList
.getDiffListBetweenSnapshots(diffIndex,
directoryDiffList.asList().size(), currentDir);
for (DirectoryDiff d : diffList) {
combined.combinePosterior(d.diff, null);
}
children = combined.apply2Current(ReadOnlyList.Util.asList(
currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID)));
children = combined.apply2Current(ReadOnlyList.Util
.asList(currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID)));
}
return children;
}
@ -383,6 +389,19 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
return NO_SNAPSHOT_ID;
}
/**
* Returns the list of diffs between two indexes corresponding to two
* snapshots.
* @param fromIndex Index of the diff corresponding to the earlier snapshot
* @param toIndex Index of the diff corresponding to the later snapshot
* @param dir The Directory to which the diffList belongs
* @return list of directory diffs
*/
List<DirectoryDiff> getDiffListBetweenSnapshots(int fromIndex, int toIndex,
INodeDirectory dir) {
return asList().getMinListForRange(fromIndex, toIndex, dir);
}
}
private static Map<INode, INode> cloneDiffList(List<INode> diffList) {
@ -597,7 +616,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
return diff != null ? diff.getChild(name, true, currentINode)
: currentINode.getChild(name, Snapshot.CURRENT_STATE_ID);
}
/** Used to record the modification of a symlink node */
public INode saveChild2Snapshot(INodeDirectory currentINode,
final INode child, final int latestSnapshotId, final INode snapshotCopy) {
@ -678,9 +697,10 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
boolean dirMetadataChanged = false;
INodeDirectoryAttributes dirCopy = null;
DiffList<DirectoryDiff> difflist = diffs.asList();
for (int i = earlierDiffIndex; i < laterDiffIndex; i++) {
DirectoryDiff sdiff = difflist.get(i);
List<DirectoryDiff> difflist = diffs
.getDiffListBetweenSnapshots(earlierDiffIndex, laterDiffIndex,
currentINode);
for (DirectoryDiff sdiff : difflist) {
diff.combinePosterior(sdiff.diff, null);
if (!dirMetadataChanged && sdiff.snapshotINode != null) {
if (dirCopy == null) {