HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable. Contributed by Jing Zhao.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605169 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Haohui Mai 2014-06-24 19:59:34 +00:00
parent 1f9a0fd927
commit bdf3de9f6b
7 changed files with 273 additions and 227 deletions

View File

@ -473,6 +473,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu) HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu)
HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable.
(Jing Zhao via wheat9)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)

View File

@ -23,16 +23,14 @@ import java.util.List;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
import com.google.common.base.Objects; import com.google.common.base.Objects;
/** /**
* This class represents to end users the difference between two snapshots of * This class represents to end users the difference between two snapshots of
* the same directory, or the difference between a snapshot of the directory and * the same directory, or the difference between a snapshot of the directory and
* its current state. Instead of capturing all the details of the diff, which * its current state. Instead of capturing all the details of the diff, this
* is stored in {@link SnapshotDiffInfo}, this class only lists where the * class only lists where the changes happened and their types.
* changes happened and their types.
*/ */
public class SnapshotDiffReport { public class SnapshotDiffReport {
private final static String LINE_SEPARATOR = System.getProperty( private final static String LINE_SEPARATOR = System.getProperty(

View File

@ -85,7 +85,18 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.util.Time.now; import static org.apache.hadoop.util.Time.now;
import java.io.*; import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.management.ManagementFactory; import java.lang.management.ManagementFactory;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.URI; import java.net.URI;
@ -170,7 +181,6 @@ import org.apache.hadoop.hdfs.protocol.RollingUpgradeException;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure; import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
@ -208,7 +218,6 @@ import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean; import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@ -7484,7 +7493,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
*/ */
SnapshotDiffReport getSnapshotDiffReport(String path, SnapshotDiffReport getSnapshotDiffReport(String path,
String fromSnapshot, String toSnapshot) throws IOException { String fromSnapshot, String toSnapshot) throws IOException {
SnapshotDiffInfo diffs = null; SnapshotDiffReport diffs;
checkOperation(OperationCategory.READ); checkOperation(OperationCategory.READ);
final FSPermissionChecker pc = getPermissionChecker(); final FSPermissionChecker pc = getPermissionChecker();
readLock(); readLock();
@ -7498,13 +7507,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
} finally { } finally {
readUnlock(); readUnlock();
} }
if (auditLog.isInfoEnabled() && isExternalInvocation()) { if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "computeSnapshotDiff", null, null, null); logAuditEvent(true, "computeSnapshotDiff", null, null, null);
} }
return diffs != null ? diffs.generateReport() : new SnapshotDiffReport( return diffs;
path, fromSnapshot, toSnapshot,
Collections.<DiffReportEntry> emptyList());
} }
private void checkSubtreeReadPermission(final FSPermissionChecker pc, private void checkSubtreeReadPermission(final FSPermissionChecker pc,

View File

@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
import java.io.DataOutput; import java.io.DataOutput;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayDeque; import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque; import java.util.Deque;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
@ -29,8 +28,6 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.server.namenode.Content; import org.apache.hadoop.hdfs.server.namenode.Content;
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
@ -41,7 +38,6 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.Quota; import org.apache.hadoop.hdfs.server.namenode.Quota;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import org.apache.hadoop.hdfs.util.Diff; import org.apache.hadoop.hdfs.util.Diff;
import org.apache.hadoop.hdfs.util.Diff.Container; import org.apache.hadoop.hdfs.util.Diff.Container;
@ -161,43 +157,6 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
} }
} }
} }
/**
* Interpret the diff and generate a list of {@link DiffReportEntry}.
* @param parentPath The relative path of the parent.
* @param fromEarlier True indicates {@code diff=later-earlier},
* False indicates {@code diff=earlier-later}
* @return A list of {@link DiffReportEntry} as the diff report.
*/
public List<DiffReportEntry> generateReport(byte[][] parentPath,
boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
List<INode> created = getList(ListType.CREATED);
List<INode> deleted = getList(ListType.DELETED);
byte[][] fullPath = new byte[parentPath.length + 1][];
System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
for (INode cnode : created) {
RenameEntry entry = renameMap.get(cnode.getId());
if (entry == null || !entry.isRename()) {
fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
: DiffType.DELETE, fullPath));
}
}
for (INode dnode : deleted) {
RenameEntry entry = renameMap.get(dnode.getId());
if (entry != null && entry.isRename()) {
list.add(new DiffReportEntry(DiffType.RENAME,
fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
} else {
fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
: DiffType.CREATE, fullPath));
}
}
return list;
}
} }
/** /**

View File

@ -21,22 +21,14 @@ import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.Content; import org.apache.hadoop.hdfs.server.namenode.Content;
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
@ -56,7 +48,6 @@ import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.primitives.SignedBytes;
/** /**
* Directories where taking snapshots is allowed. * Directories where taking snapshots is allowed.
@ -79,164 +70,6 @@ public class INodeDirectorySnapshottable extends INodeDirectory {
} }
return (INodeDirectorySnapshottable)dir; return (INodeDirectorySnapshottable)dir;
} }
/**
* A class describing the difference between snapshots of a snapshottable
* directory.
*/
public static class SnapshotDiffInfo {
/** Compare two inodes based on their full names */
public static final Comparator<INode> INODE_COMPARATOR =
new Comparator<INode>() {
@Override
public int compare(INode left, INode right) {
if (left == null) {
return right == null ? 0 : -1;
} else {
if (right == null) {
return 1;
} else {
int cmp = compare(left.getParent(), right.getParent());
return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
}
}
}
};
static class RenameEntry {
private byte[][] sourcePath;
private byte[][] targetPath;
void setSource(INode source, byte[][] sourceParentPath) {
Preconditions.checkState(sourcePath == null);
sourcePath = new byte[sourceParentPath.length + 1][];
System.arraycopy(sourceParentPath, 0, sourcePath, 0,
sourceParentPath.length);
sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
}
void setTarget(INode target, byte[][] targetParentPath) {
targetPath = new byte[targetParentPath.length + 1][];
System.arraycopy(targetParentPath, 0, targetPath, 0,
targetParentPath.length);
targetPath[targetPath.length - 1] = target.getLocalNameBytes();
}
void setTarget(byte[][] targetPath) {
this.targetPath = targetPath;
}
boolean isRename() {
return sourcePath != null && targetPath != null;
}
byte[][] getSourcePath() {
return sourcePath;
}
byte[][] getTargetPath() {
return targetPath;
}
}
/** The root directory of the snapshots */
private final INodeDirectorySnapshottable snapshotRoot;
/** The starting point of the difference */
private final Snapshot from;
/** The end point of the difference */
private final Snapshot to;
/**
* A map recording modified INodeFile and INodeDirectory and their relative
* path corresponding to the snapshot root. Sorted based on their names.
*/
private final SortedMap<INode, byte[][]> diffMap =
new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
/**
* A map capturing the detailed difference about file creation/deletion.
* Each key indicates a directory whose children have been changed between
* the two snapshots, while its associated value is a {@link ChildrenDiff}
* storing the changes (creation/deletion) happened to the children (files).
*/
private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
new HashMap<INodeDirectory, ChildrenDiff>();
private final Map<Long, RenameEntry> renameMap =
new HashMap<Long, RenameEntry>();
SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
Snapshot end) {
this.snapshotRoot = snapshotRoot;
this.from = start;
this.to = end;
}
/** Add a dir-diff pair */
private void addDirDiff(INodeDirectory dir, byte[][] relativePath,
ChildrenDiff diff) {
dirDiffMap.put(dir, diff);
diffMap.put(dir, relativePath);
// detect rename
for (INode created : diff.getList(ListType.CREATED)) {
if (created.isReference()) {
RenameEntry entry = getEntry(created.getId());
if (entry.getTargetPath() == null) {
entry.setTarget(created, relativePath);
}
}
}
for (INode deleted : diff.getList(ListType.DELETED)) {
if (deleted instanceof INodeReference.WithName) {
RenameEntry entry = getEntry(deleted.getId());
entry.setSource(deleted, relativePath);
}
}
}
private RenameEntry getEntry(long inodeId) {
RenameEntry entry = renameMap.get(inodeId);
if (entry == null) {
entry = new RenameEntry();
renameMap.put(inodeId, entry);
}
return entry;
}
private void setRenameTarget(long inodeId, byte[][] path) {
getEntry(inodeId).setTarget(path);
}
/** Add a modified file */
private void addFileDiff(INodeFile file, byte[][] relativePath) {
diffMap.put(file, relativePath);
}
/** @return True if {@link #from} is earlier than {@link #to} */
private boolean isFromEarlier() {
return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
}
/**
* Generate a {@link SnapshotDiffReport} based on detailed diff information.
* @return A {@link SnapshotDiffReport} describing the difference
*/
public SnapshotDiffReport generateReport() {
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
for (INode node : diffMap.keySet()) {
diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
.get(node), null));
if (node.isDirectory()) {
ChildrenDiff dirDiff = dirDiffMap.get(node);
List<DiffReportEntry> subList = dirDiff.generateReport(
diffMap.get(node), isFromEarlier(), renameMap);
diffReportList.addAll(subList);
}
}
return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
diffReportList);
}
}
/** /**
* Snapshots of this directory in ascending order of snapshot names. * Snapshots of this directory in ascending order of snapshot names.
@ -496,9 +329,9 @@ public class INodeDirectorySnapshottable extends INodeDirectory {
private void computeDiffRecursively(INode node, List<byte[]> parentPath, private void computeDiffRecursively(INode node, List<byte[]> parentPath,
SnapshotDiffInfo diffReport) { SnapshotDiffInfo diffReport) {
final Snapshot earlierSnapshot = diffReport.isFromEarlier() ? final Snapshot earlierSnapshot = diffReport.isFromEarlier() ?
diffReport.from : diffReport.to; diffReport.getFrom() : diffReport.getTo();
final Snapshot laterSnapshot = diffReport.isFromEarlier() ? final Snapshot laterSnapshot = diffReport.isFromEarlier() ?
diffReport.to : diffReport.from; diffReport.getTo() : diffReport.getFrom();
byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]); byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
if (node.isDirectory()) { if (node.isDirectory()) {
final ChildrenDiff diff = new ChildrenDiff(); final ChildrenDiff diff = new ChildrenDiff();

View File

@ -0,0 +1,242 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
import org.apache.hadoop.hdfs.util.Diff.ListType;
import com.google.common.base.Preconditions;
import com.google.common.primitives.SignedBytes;
/**
* A class describing the difference between snapshots of a snapshottable
* directory.
*/
class SnapshotDiffInfo {
/** Compare two inodes based on their full names */
public static final Comparator<INode> INODE_COMPARATOR =
new Comparator<INode>() {
@Override
public int compare(INode left, INode right) {
if (left == null) {
return right == null ? 0 : -1;
} else {
if (right == null) {
return 1;
} else {
int cmp = compare(left.getParent(), right.getParent());
return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
}
}
}
};
static class RenameEntry {
private byte[][] sourcePath;
private byte[][] targetPath;
void setSource(INode source, byte[][] sourceParentPath) {
Preconditions.checkState(sourcePath == null);
sourcePath = new byte[sourceParentPath.length + 1][];
System.arraycopy(sourceParentPath, 0, sourcePath, 0,
sourceParentPath.length);
sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
}
void setTarget(INode target, byte[][] targetParentPath) {
targetPath = new byte[targetParentPath.length + 1][];
System.arraycopy(targetParentPath, 0, targetPath, 0,
targetParentPath.length);
targetPath[targetPath.length - 1] = target.getLocalNameBytes();
}
void setTarget(byte[][] targetPath) {
this.targetPath = targetPath;
}
boolean isRename() {
return sourcePath != null && targetPath != null;
}
byte[][] getSourcePath() {
return sourcePath;
}
byte[][] getTargetPath() {
return targetPath;
}
}
/** The root directory of the snapshots */
private final INodeDirectorySnapshottable snapshotRoot;
/** The starting point of the difference */
private final Snapshot from;
/** The end point of the difference */
private final Snapshot to;
/**
* A map recording modified INodeFile and INodeDirectory and their relative
* path corresponding to the snapshot root. Sorted based on their names.
*/
private final SortedMap<INode, byte[][]> diffMap =
new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
/**
* A map capturing the detailed difference about file creation/deletion.
* Each key indicates a directory whose children have been changed between
* the two snapshots, while its associated value is a {@link ChildrenDiff}
* storing the changes (creation/deletion) happened to the children (files).
*/
private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
new HashMap<INodeDirectory, ChildrenDiff>();
private final Map<Long, RenameEntry> renameMap =
new HashMap<Long, RenameEntry>();
SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
Snapshot end) {
this.snapshotRoot = snapshotRoot;
this.from = start;
this.to = end;
}
/** Add a dir-diff pair */
void addDirDiff(INodeDirectory dir, byte[][] relativePath, ChildrenDiff diff) {
dirDiffMap.put(dir, diff);
diffMap.put(dir, relativePath);
// detect rename
for (INode created : diff.getList(ListType.CREATED)) {
if (created.isReference()) {
RenameEntry entry = getEntry(created.getId());
if (entry.getTargetPath() == null) {
entry.setTarget(created, relativePath);
}
}
}
for (INode deleted : diff.getList(ListType.DELETED)) {
if (deleted instanceof INodeReference.WithName) {
RenameEntry entry = getEntry(deleted.getId());
entry.setSource(deleted, relativePath);
}
}
}
Snapshot getFrom() {
return from;
}
Snapshot getTo() {
return to;
}
private RenameEntry getEntry(long inodeId) {
RenameEntry entry = renameMap.get(inodeId);
if (entry == null) {
entry = new RenameEntry();
renameMap.put(inodeId, entry);
}
return entry;
}
void setRenameTarget(long inodeId, byte[][] path) {
getEntry(inodeId).setTarget(path);
}
/** Add a modified file */
void addFileDiff(INodeFile file, byte[][] relativePath) {
diffMap.put(file, relativePath);
}
/** @return True if {@link #from} is earlier than {@link #to} */
boolean isFromEarlier() {
return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
}
/**
* Generate a {@link SnapshotDiffReport} based on detailed diff information.
* @return A {@link SnapshotDiffReport} describing the difference
*/
public SnapshotDiffReport generateReport() {
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
for (INode node : diffMap.keySet()) {
diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
.get(node), null));
if (node.isDirectory()) {
List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node),
diffMap.get(node), isFromEarlier(), renameMap);
diffReportList.addAll(subList);
}
}
return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
diffReportList);
}
/**
* Interpret the ChildrenDiff and generate a list of {@link DiffReportEntry}.
* @param dirDiff The ChildrenDiff.
* @param parentPath The relative path of the parent.
* @param fromEarlier True indicates {@code diff=later-earlier},
* False indicates {@code diff=earlier-later}
* @param renameMap A map containing information about rename operations.
* @return A list of {@link DiffReportEntry} as the diff report.
*/
private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff,
byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
List<INode> created = dirDiff.getList(ListType.CREATED);
List<INode> deleted = dirDiff.getList(ListType.DELETED);
byte[][] fullPath = new byte[parentPath.length + 1][];
System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
for (INode cnode : created) {
RenameEntry entry = renameMap.get(cnode.getId());
if (entry == null || !entry.isRename()) {
fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
: DiffType.DELETE, fullPath));
}
}
for (INode dnode : deleted) {
RenameEntry entry = renameMap.get(dnode.getId());
if (entry != null && entry.isRename()) {
list.add(new DiffReportEntry(DiffType.RENAME,
fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
} else {
fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
: DiffType.CREATE, fullPath));
}
}
return list;
}
}

View File

@ -30,9 +30,11 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.management.ObjectName; import javax.management.ObjectName;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.protocol.SnapshotInfo; import org.apache.hadoop.hdfs.protocol.SnapshotInfo;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@ -40,7 +42,6 @@ import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodesInPath; import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.metrics2.util.MBeans;
/** /**
@ -361,12 +362,13 @@ public class SnapshotManager implements SnapshotStatsMXBean {
* Compute the difference between two snapshots of a directory, or between a * Compute the difference between two snapshots of a directory, or between a
* snapshot of the directory and its current tree. * snapshot of the directory and its current tree.
*/ */
public SnapshotDiffInfo diff(final String path, final String from, public SnapshotDiffReport diff(final String path, final String from,
final String to) throws IOException { final String to) throws IOException {
if ((from == null || from.isEmpty()) if ((from == null || from.isEmpty())
&& (to == null || to.isEmpty())) { && (to == null || to.isEmpty())) {
// both fromSnapshot and toSnapshot indicate the current tree // both fromSnapshot and toSnapshot indicate the current tree
return null; return new SnapshotDiffReport(path, from, to,
Collections.<DiffReportEntry> emptyList());
} }
// Find the source root directory path where the snapshots were taken. // Find the source root directory path where the snapshots were taken.
@ -374,8 +376,10 @@ public class SnapshotManager implements SnapshotStatsMXBean {
INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString()); INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString());
final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable
.valueOf(inodesInPath.getLastINode(), path); .valueOf(inodesInPath.getLastINode(), path);
return snapshotRoot.computeDiff(from, to); final SnapshotDiffInfo diffs = snapshotRoot.computeDiff(from, to);
return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
path, from, to, Collections.<DiffReportEntry> emptyList());
} }
public void clearSnapshottableDirs() { public void clearSnapshottableDirs() {