HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable. Contributed by Jing Zhao.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605169 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
1f9a0fd927
commit
bdf3de9f6b
|
@ -473,6 +473,9 @@ Release 2.5.0 - UNRELEASED
|
||||||
|
|
||||||
HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu)
|
HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu)
|
||||||
|
|
||||||
|
HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable.
|
||||||
|
(Jing Zhao via wheat9)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
|
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
|
||||||
|
|
|
@ -23,16 +23,14 @@ import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.DFSUtil;
|
import org.apache.hadoop.hdfs.DFSUtil;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
|
|
||||||
|
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class represents to end users the difference between two snapshots of
|
* This class represents to end users the difference between two snapshots of
|
||||||
* the same directory, or the difference between a snapshot of the directory and
|
* the same directory, or the difference between a snapshot of the directory and
|
||||||
* its current state. Instead of capturing all the details of the diff, which
|
* its current state. Instead of capturing all the details of the diff, this
|
||||||
* is stored in {@link SnapshotDiffInfo}, this class only lists where the
|
* class only lists where the changes happened and their types.
|
||||||
* changes happened and their types.
|
|
||||||
*/
|
*/
|
||||||
public class SnapshotDiffReport {
|
public class SnapshotDiffReport {
|
||||||
private final static String LINE_SEPARATOR = System.getProperty(
|
private final static String LINE_SEPARATOR = System.getProperty(
|
||||||
|
|
|
@ -85,7 +85,18 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
|
||||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
|
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
|
||||||
import static org.apache.hadoop.util.Time.now;
|
import static org.apache.hadoop.util.Time.now;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.BufferedWriter;
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.DataInput;
|
||||||
|
import java.io.DataInputStream;
|
||||||
|
import java.io.DataOutputStream;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.io.StringWriter;
|
||||||
import java.lang.management.ManagementFactory;
|
import java.lang.management.ManagementFactory;
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
@ -170,7 +181,6 @@ import org.apache.hadoop.hdfs.protocol.RollingUpgradeException;
|
||||||
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
|
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
|
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
|
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
|
||||||
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
|
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
|
||||||
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
|
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
|
||||||
|
@ -208,7 +218,6 @@ import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
|
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
|
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
|
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
|
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
|
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
|
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
|
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
|
||||||
|
@ -7484,7 +7493,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
||||||
*/
|
*/
|
||||||
SnapshotDiffReport getSnapshotDiffReport(String path,
|
SnapshotDiffReport getSnapshotDiffReport(String path,
|
||||||
String fromSnapshot, String toSnapshot) throws IOException {
|
String fromSnapshot, String toSnapshot) throws IOException {
|
||||||
SnapshotDiffInfo diffs = null;
|
SnapshotDiffReport diffs;
|
||||||
checkOperation(OperationCategory.READ);
|
checkOperation(OperationCategory.READ);
|
||||||
final FSPermissionChecker pc = getPermissionChecker();
|
final FSPermissionChecker pc = getPermissionChecker();
|
||||||
readLock();
|
readLock();
|
||||||
|
@ -7498,13 +7507,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
||||||
} finally {
|
} finally {
|
||||||
readUnlock();
|
readUnlock();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
|
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
|
||||||
logAuditEvent(true, "computeSnapshotDiff", null, null, null);
|
logAuditEvent(true, "computeSnapshotDiff", null, null, null);
|
||||||
}
|
}
|
||||||
return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
|
return diffs;
|
||||||
path, fromSnapshot, toSnapshot,
|
|
||||||
Collections.<DiffReportEntry> emptyList());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkSubtreeReadPermission(final FSPermissionChecker pc,
|
private void checkSubtreeReadPermission(final FSPermissionChecker pc,
|
||||||
|
|
|
@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
|
||||||
import java.io.DataOutput;
|
import java.io.DataOutput;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayDeque;
|
import java.util.ArrayDeque;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Deque;
|
import java.util.Deque;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
|
@ -29,8 +28,6 @@ import java.util.Map;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
|
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
|
|
||||||
import org.apache.hadoop.hdfs.server.namenode.Content;
|
import org.apache.hadoop.hdfs.server.namenode.Content;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
|
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
|
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
|
||||||
|
@ -41,7 +38,6 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
|
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.Quota;
|
import org.apache.hadoop.hdfs.server.namenode.Quota;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry;
|
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
|
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
|
||||||
import org.apache.hadoop.hdfs.util.Diff;
|
import org.apache.hadoop.hdfs.util.Diff;
|
||||||
import org.apache.hadoop.hdfs.util.Diff.Container;
|
import org.apache.hadoop.hdfs.util.Diff.Container;
|
||||||
|
@ -161,43 +157,6 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Interpret the diff and generate a list of {@link DiffReportEntry}.
|
|
||||||
* @param parentPath The relative path of the parent.
|
|
||||||
* @param fromEarlier True indicates {@code diff=later-earlier},
|
|
||||||
* False indicates {@code diff=earlier-later}
|
|
||||||
* @return A list of {@link DiffReportEntry} as the diff report.
|
|
||||||
*/
|
|
||||||
public List<DiffReportEntry> generateReport(byte[][] parentPath,
|
|
||||||
boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
|
|
||||||
List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
|
|
||||||
List<INode> created = getList(ListType.CREATED);
|
|
||||||
List<INode> deleted = getList(ListType.DELETED);
|
|
||||||
byte[][] fullPath = new byte[parentPath.length + 1][];
|
|
||||||
System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
|
|
||||||
for (INode cnode : created) {
|
|
||||||
RenameEntry entry = renameMap.get(cnode.getId());
|
|
||||||
if (entry == null || !entry.isRename()) {
|
|
||||||
fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
|
|
||||||
list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
|
|
||||||
: DiffType.DELETE, fullPath));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (INode dnode : deleted) {
|
|
||||||
RenameEntry entry = renameMap.get(dnode.getId());
|
|
||||||
if (entry != null && entry.isRename()) {
|
|
||||||
list.add(new DiffReportEntry(DiffType.RENAME,
|
|
||||||
fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
|
|
||||||
fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
|
|
||||||
} else {
|
|
||||||
fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
|
|
||||||
list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
|
|
||||||
: DiffType.CREATE, fullPath));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return list;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -21,22 +21,14 @@ import java.io.IOException;
|
||||||
import java.io.PrintWriter;
|
import java.io.PrintWriter;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.LinkedList;
|
import java.util.LinkedList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.SortedMap;
|
|
||||||
import java.util.TreeMap;
|
|
||||||
|
|
||||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.hdfs.DFSUtil;
|
import org.apache.hadoop.hdfs.DFSUtil;
|
||||||
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
|
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotException;
|
import org.apache.hadoop.hdfs.protocol.SnapshotException;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.Content;
|
import org.apache.hadoop.hdfs.server.namenode.Content;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
|
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
|
||||||
|
@ -56,7 +48,6 @@ import org.apache.hadoop.util.Time;
|
||||||
|
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.google.common.primitives.SignedBytes;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Directories where taking snapshots is allowed.
|
* Directories where taking snapshots is allowed.
|
||||||
|
@ -79,164 +70,6 @@ public class INodeDirectorySnapshottable extends INodeDirectory {
|
||||||
}
|
}
|
||||||
return (INodeDirectorySnapshottable)dir;
|
return (INodeDirectorySnapshottable)dir;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* A class describing the difference between snapshots of a snapshottable
|
|
||||||
* directory.
|
|
||||||
*/
|
|
||||||
public static class SnapshotDiffInfo {
|
|
||||||
/** Compare two inodes based on their full names */
|
|
||||||
public static final Comparator<INode> INODE_COMPARATOR =
|
|
||||||
new Comparator<INode>() {
|
|
||||||
@Override
|
|
||||||
public int compare(INode left, INode right) {
|
|
||||||
if (left == null) {
|
|
||||||
return right == null ? 0 : -1;
|
|
||||||
} else {
|
|
||||||
if (right == null) {
|
|
||||||
return 1;
|
|
||||||
} else {
|
|
||||||
int cmp = compare(left.getParent(), right.getParent());
|
|
||||||
return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
|
|
||||||
left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
static class RenameEntry {
|
|
||||||
private byte[][] sourcePath;
|
|
||||||
private byte[][] targetPath;
|
|
||||||
|
|
||||||
void setSource(INode source, byte[][] sourceParentPath) {
|
|
||||||
Preconditions.checkState(sourcePath == null);
|
|
||||||
sourcePath = new byte[sourceParentPath.length + 1][];
|
|
||||||
System.arraycopy(sourceParentPath, 0, sourcePath, 0,
|
|
||||||
sourceParentPath.length);
|
|
||||||
sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
|
|
||||||
}
|
|
||||||
|
|
||||||
void setTarget(INode target, byte[][] targetParentPath) {
|
|
||||||
targetPath = new byte[targetParentPath.length + 1][];
|
|
||||||
System.arraycopy(targetParentPath, 0, targetPath, 0,
|
|
||||||
targetParentPath.length);
|
|
||||||
targetPath[targetPath.length - 1] = target.getLocalNameBytes();
|
|
||||||
}
|
|
||||||
|
|
||||||
void setTarget(byte[][] targetPath) {
|
|
||||||
this.targetPath = targetPath;
|
|
||||||
}
|
|
||||||
|
|
||||||
boolean isRename() {
|
|
||||||
return sourcePath != null && targetPath != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
byte[][] getSourcePath() {
|
|
||||||
return sourcePath;
|
|
||||||
}
|
|
||||||
|
|
||||||
byte[][] getTargetPath() {
|
|
||||||
return targetPath;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The root directory of the snapshots */
|
|
||||||
private final INodeDirectorySnapshottable snapshotRoot;
|
|
||||||
/** The starting point of the difference */
|
|
||||||
private final Snapshot from;
|
|
||||||
/** The end point of the difference */
|
|
||||||
private final Snapshot to;
|
|
||||||
/**
|
|
||||||
* A map recording modified INodeFile and INodeDirectory and their relative
|
|
||||||
* path corresponding to the snapshot root. Sorted based on their names.
|
|
||||||
*/
|
|
||||||
private final SortedMap<INode, byte[][]> diffMap =
|
|
||||||
new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
|
|
||||||
/**
|
|
||||||
* A map capturing the detailed difference about file creation/deletion.
|
|
||||||
* Each key indicates a directory whose children have been changed between
|
|
||||||
* the two snapshots, while its associated value is a {@link ChildrenDiff}
|
|
||||||
* storing the changes (creation/deletion) happened to the children (files).
|
|
||||||
*/
|
|
||||||
private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
|
|
||||||
new HashMap<INodeDirectory, ChildrenDiff>();
|
|
||||||
|
|
||||||
private final Map<Long, RenameEntry> renameMap =
|
|
||||||
new HashMap<Long, RenameEntry>();
|
|
||||||
|
|
||||||
SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
|
|
||||||
Snapshot end) {
|
|
||||||
this.snapshotRoot = snapshotRoot;
|
|
||||||
this.from = start;
|
|
||||||
this.to = end;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Add a dir-diff pair */
|
|
||||||
private void addDirDiff(INodeDirectory dir, byte[][] relativePath,
|
|
||||||
ChildrenDiff diff) {
|
|
||||||
dirDiffMap.put(dir, diff);
|
|
||||||
diffMap.put(dir, relativePath);
|
|
||||||
// detect rename
|
|
||||||
for (INode created : diff.getList(ListType.CREATED)) {
|
|
||||||
if (created.isReference()) {
|
|
||||||
RenameEntry entry = getEntry(created.getId());
|
|
||||||
if (entry.getTargetPath() == null) {
|
|
||||||
entry.setTarget(created, relativePath);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (INode deleted : diff.getList(ListType.DELETED)) {
|
|
||||||
if (deleted instanceof INodeReference.WithName) {
|
|
||||||
RenameEntry entry = getEntry(deleted.getId());
|
|
||||||
entry.setSource(deleted, relativePath);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private RenameEntry getEntry(long inodeId) {
|
|
||||||
RenameEntry entry = renameMap.get(inodeId);
|
|
||||||
if (entry == null) {
|
|
||||||
entry = new RenameEntry();
|
|
||||||
renameMap.put(inodeId, entry);
|
|
||||||
}
|
|
||||||
return entry;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void setRenameTarget(long inodeId, byte[][] path) {
|
|
||||||
getEntry(inodeId).setTarget(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Add a modified file */
|
|
||||||
private void addFileDiff(INodeFile file, byte[][] relativePath) {
|
|
||||||
diffMap.put(file, relativePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return True if {@link #from} is earlier than {@link #to} */
|
|
||||||
private boolean isFromEarlier() {
|
|
||||||
return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate a {@link SnapshotDiffReport} based on detailed diff information.
|
|
||||||
* @return A {@link SnapshotDiffReport} describing the difference
|
|
||||||
*/
|
|
||||||
public SnapshotDiffReport generateReport() {
|
|
||||||
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
|
|
||||||
for (INode node : diffMap.keySet()) {
|
|
||||||
diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
|
|
||||||
.get(node), null));
|
|
||||||
if (node.isDirectory()) {
|
|
||||||
ChildrenDiff dirDiff = dirDiffMap.get(node);
|
|
||||||
List<DiffReportEntry> subList = dirDiff.generateReport(
|
|
||||||
diffMap.get(node), isFromEarlier(), renameMap);
|
|
||||||
diffReportList.addAll(subList);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
|
|
||||||
Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
|
|
||||||
diffReportList);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Snapshots of this directory in ascending order of snapshot names.
|
* Snapshots of this directory in ascending order of snapshot names.
|
||||||
|
@ -496,9 +329,9 @@ public class INodeDirectorySnapshottable extends INodeDirectory {
|
||||||
private void computeDiffRecursively(INode node, List<byte[]> parentPath,
|
private void computeDiffRecursively(INode node, List<byte[]> parentPath,
|
||||||
SnapshotDiffInfo diffReport) {
|
SnapshotDiffInfo diffReport) {
|
||||||
final Snapshot earlierSnapshot = diffReport.isFromEarlier() ?
|
final Snapshot earlierSnapshot = diffReport.isFromEarlier() ?
|
||||||
diffReport.from : diffReport.to;
|
diffReport.getFrom() : diffReport.getTo();
|
||||||
final Snapshot laterSnapshot = diffReport.isFromEarlier() ?
|
final Snapshot laterSnapshot = diffReport.isFromEarlier() ?
|
||||||
diffReport.to : diffReport.from;
|
diffReport.getTo() : diffReport.getFrom();
|
||||||
byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
|
byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
|
||||||
if (node.isDirectory()) {
|
if (node.isDirectory()) {
|
||||||
final ChildrenDiff diff = new ChildrenDiff();
|
final ChildrenDiff diff = new ChildrenDiff();
|
||||||
|
|
|
@ -0,0 +1,242 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hdfs.server.namenode.snapshot;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.SortedMap;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
|
||||||
|
import org.apache.hadoop.hdfs.server.namenode.INode;
|
||||||
|
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
|
||||||
|
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
||||||
|
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
|
||||||
|
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
|
||||||
|
import org.apache.hadoop.hdfs.util.Diff.ListType;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
import com.google.common.primitives.SignedBytes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A class describing the difference between snapshots of a snapshottable
|
||||||
|
* directory.
|
||||||
|
*/
|
||||||
|
class SnapshotDiffInfo {
|
||||||
|
/** Compare two inodes based on their full names */
|
||||||
|
public static final Comparator<INode> INODE_COMPARATOR =
|
||||||
|
new Comparator<INode>() {
|
||||||
|
@Override
|
||||||
|
public int compare(INode left, INode right) {
|
||||||
|
if (left == null) {
|
||||||
|
return right == null ? 0 : -1;
|
||||||
|
} else {
|
||||||
|
if (right == null) {
|
||||||
|
return 1;
|
||||||
|
} else {
|
||||||
|
int cmp = compare(left.getParent(), right.getParent());
|
||||||
|
return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
|
||||||
|
left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
static class RenameEntry {
|
||||||
|
private byte[][] sourcePath;
|
||||||
|
private byte[][] targetPath;
|
||||||
|
|
||||||
|
void setSource(INode source, byte[][] sourceParentPath) {
|
||||||
|
Preconditions.checkState(sourcePath == null);
|
||||||
|
sourcePath = new byte[sourceParentPath.length + 1][];
|
||||||
|
System.arraycopy(sourceParentPath, 0, sourcePath, 0,
|
||||||
|
sourceParentPath.length);
|
||||||
|
sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
|
||||||
|
}
|
||||||
|
|
||||||
|
void setTarget(INode target, byte[][] targetParentPath) {
|
||||||
|
targetPath = new byte[targetParentPath.length + 1][];
|
||||||
|
System.arraycopy(targetParentPath, 0, targetPath, 0,
|
||||||
|
targetParentPath.length);
|
||||||
|
targetPath[targetPath.length - 1] = target.getLocalNameBytes();
|
||||||
|
}
|
||||||
|
|
||||||
|
void setTarget(byte[][] targetPath) {
|
||||||
|
this.targetPath = targetPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean isRename() {
|
||||||
|
return sourcePath != null && targetPath != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
byte[][] getSourcePath() {
|
||||||
|
return sourcePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
byte[][] getTargetPath() {
|
||||||
|
return targetPath;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The root directory of the snapshots */
|
||||||
|
private final INodeDirectorySnapshottable snapshotRoot;
|
||||||
|
/** The starting point of the difference */
|
||||||
|
private final Snapshot from;
|
||||||
|
/** The end point of the difference */
|
||||||
|
private final Snapshot to;
|
||||||
|
/**
|
||||||
|
* A map recording modified INodeFile and INodeDirectory and their relative
|
||||||
|
* path corresponding to the snapshot root. Sorted based on their names.
|
||||||
|
*/
|
||||||
|
private final SortedMap<INode, byte[][]> diffMap =
|
||||||
|
new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
|
||||||
|
/**
|
||||||
|
* A map capturing the detailed difference about file creation/deletion.
|
||||||
|
* Each key indicates a directory whose children have been changed between
|
||||||
|
* the two snapshots, while its associated value is a {@link ChildrenDiff}
|
||||||
|
* storing the changes (creation/deletion) happened to the children (files).
|
||||||
|
*/
|
||||||
|
private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
|
||||||
|
new HashMap<INodeDirectory, ChildrenDiff>();
|
||||||
|
|
||||||
|
private final Map<Long, RenameEntry> renameMap =
|
||||||
|
new HashMap<Long, RenameEntry>();
|
||||||
|
|
||||||
|
SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
|
||||||
|
Snapshot end) {
|
||||||
|
this.snapshotRoot = snapshotRoot;
|
||||||
|
this.from = start;
|
||||||
|
this.to = end;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Add a dir-diff pair */
|
||||||
|
void addDirDiff(INodeDirectory dir, byte[][] relativePath, ChildrenDiff diff) {
|
||||||
|
dirDiffMap.put(dir, diff);
|
||||||
|
diffMap.put(dir, relativePath);
|
||||||
|
// detect rename
|
||||||
|
for (INode created : diff.getList(ListType.CREATED)) {
|
||||||
|
if (created.isReference()) {
|
||||||
|
RenameEntry entry = getEntry(created.getId());
|
||||||
|
if (entry.getTargetPath() == null) {
|
||||||
|
entry.setTarget(created, relativePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (INode deleted : diff.getList(ListType.DELETED)) {
|
||||||
|
if (deleted instanceof INodeReference.WithName) {
|
||||||
|
RenameEntry entry = getEntry(deleted.getId());
|
||||||
|
entry.setSource(deleted, relativePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Snapshot getFrom() {
|
||||||
|
return from;
|
||||||
|
}
|
||||||
|
|
||||||
|
Snapshot getTo() {
|
||||||
|
return to;
|
||||||
|
}
|
||||||
|
|
||||||
|
private RenameEntry getEntry(long inodeId) {
|
||||||
|
RenameEntry entry = renameMap.get(inodeId);
|
||||||
|
if (entry == null) {
|
||||||
|
entry = new RenameEntry();
|
||||||
|
renameMap.put(inodeId, entry);
|
||||||
|
}
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
void setRenameTarget(long inodeId, byte[][] path) {
|
||||||
|
getEntry(inodeId).setTarget(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Add a modified file */
|
||||||
|
void addFileDiff(INodeFile file, byte[][] relativePath) {
|
||||||
|
diffMap.put(file, relativePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return True if {@link #from} is earlier than {@link #to} */
|
||||||
|
boolean isFromEarlier() {
|
||||||
|
return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a {@link SnapshotDiffReport} based on detailed diff information.
|
||||||
|
* @return A {@link SnapshotDiffReport} describing the difference
|
||||||
|
*/
|
||||||
|
public SnapshotDiffReport generateReport() {
|
||||||
|
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
|
||||||
|
for (INode node : diffMap.keySet()) {
|
||||||
|
diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
|
||||||
|
.get(node), null));
|
||||||
|
if (node.isDirectory()) {
|
||||||
|
List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node),
|
||||||
|
diffMap.get(node), isFromEarlier(), renameMap);
|
||||||
|
diffReportList.addAll(subList);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
|
||||||
|
Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
|
||||||
|
diffReportList);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Interpret the ChildrenDiff and generate a list of {@link DiffReportEntry}.
|
||||||
|
* @param dirDiff The ChildrenDiff.
|
||||||
|
* @param parentPath The relative path of the parent.
|
||||||
|
* @param fromEarlier True indicates {@code diff=later-earlier},
|
||||||
|
* False indicates {@code diff=earlier-later}
|
||||||
|
* @param renameMap A map containing information about rename operations.
|
||||||
|
* @return A list of {@link DiffReportEntry} as the diff report.
|
||||||
|
*/
|
||||||
|
private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff,
|
||||||
|
byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
|
||||||
|
List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
|
||||||
|
List<INode> created = dirDiff.getList(ListType.CREATED);
|
||||||
|
List<INode> deleted = dirDiff.getList(ListType.DELETED);
|
||||||
|
byte[][] fullPath = new byte[parentPath.length + 1][];
|
||||||
|
System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
|
||||||
|
for (INode cnode : created) {
|
||||||
|
RenameEntry entry = renameMap.get(cnode.getId());
|
||||||
|
if (entry == null || !entry.isRename()) {
|
||||||
|
fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
|
||||||
|
list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
|
||||||
|
: DiffType.DELETE, fullPath));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (INode dnode : deleted) {
|
||||||
|
RenameEntry entry = renameMap.get(dnode.getId());
|
||||||
|
if (entry != null && entry.isRename()) {
|
||||||
|
list.add(new DiffReportEntry(DiffType.RENAME,
|
||||||
|
fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
|
||||||
|
fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
|
||||||
|
} else {
|
||||||
|
fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
|
||||||
|
list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
|
||||||
|
: DiffType.CREATE, fullPath));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
}
|
|
@ -30,9 +30,11 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
import javax.management.ObjectName;
|
import javax.management.ObjectName;
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.DFSUtil;
|
import org.apache.hadoop.hdfs.DFSUtil;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotException;
|
import org.apache.hadoop.hdfs.protocol.SnapshotException;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotInfo;
|
import org.apache.hadoop.hdfs.protocol.SnapshotInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
|
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
|
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
|
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
|
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
|
||||||
|
@ -40,7 +42,6 @@ import org.apache.hadoop.hdfs.server.namenode.INode;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
|
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
|
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
|
import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
|
|
||||||
import org.apache.hadoop.metrics2.util.MBeans;
|
import org.apache.hadoop.metrics2.util.MBeans;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -361,12 +362,13 @@ public class SnapshotManager implements SnapshotStatsMXBean {
|
||||||
* Compute the difference between two snapshots of a directory, or between a
|
* Compute the difference between two snapshots of a directory, or between a
|
||||||
* snapshot of the directory and its current tree.
|
* snapshot of the directory and its current tree.
|
||||||
*/
|
*/
|
||||||
public SnapshotDiffInfo diff(final String path, final String from,
|
public SnapshotDiffReport diff(final String path, final String from,
|
||||||
final String to) throws IOException {
|
final String to) throws IOException {
|
||||||
if ((from == null || from.isEmpty())
|
if ((from == null || from.isEmpty())
|
||||||
&& (to == null || to.isEmpty())) {
|
&& (to == null || to.isEmpty())) {
|
||||||
// both fromSnapshot and toSnapshot indicate the current tree
|
// both fromSnapshot and toSnapshot indicate the current tree
|
||||||
return null;
|
return new SnapshotDiffReport(path, from, to,
|
||||||
|
Collections.<DiffReportEntry> emptyList());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find the source root directory path where the snapshots were taken.
|
// Find the source root directory path where the snapshots were taken.
|
||||||
|
@ -374,8 +376,10 @@ public class SnapshotManager implements SnapshotStatsMXBean {
|
||||||
INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString());
|
INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString());
|
||||||
final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable
|
final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable
|
||||||
.valueOf(inodesInPath.getLastINode(), path);
|
.valueOf(inodesInPath.getLastINode(), path);
|
||||||
|
|
||||||
return snapshotRoot.computeDiff(from, to);
|
final SnapshotDiffInfo diffs = snapshotRoot.computeDiff(from, to);
|
||||||
|
return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
|
||||||
|
path, from, to, Collections.<DiffReportEntry> emptyList());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void clearSnapshottableDirs() {
|
public void clearSnapshottableDirs() {
|
||||||
|
|
Loading…
Reference in New Issue