diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index aaef38dd0cd..43d3bd6febf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -219,6 +219,9 @@ Release 2.5.0 - UNRELEASED HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu) + HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable. + (Jing Zhao via wheat9) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index ba96371ffa5..b0db838dbda 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -23,16 +23,14 @@ import java.util.List; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo; import com.google.common.base.Objects; /** * This class represents to end users the difference between two snapshots of * the same directory, or the difference between a snapshot of the directory and - * its current state. Instead of capturing all the details of the diff, which - * is stored in {@link SnapshotDiffInfo}, this class only lists where the - * changes happened and their types. + * its current state. Instead of capturing all the details of the diff, this + * class only lists where the changes happened and their types. */ public class SnapshotDiffReport { private final static String LINE_SEPARATOR = System.getProperty( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index df5d0dbba9b..f929eaf6e37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -87,7 +87,18 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SUPPORT_APPEND_KEY; import static org.apache.hadoop.util.Time.now; -import java.io.*; +import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.StringWriter; import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.URI; @@ -173,7 +184,6 @@ import org.apache.hadoop.hdfs.protocol.RollingUpgradeException; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; @@ -211,7 +221,6 @@ import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer; import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; @@ -7483,7 +7492,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, */ SnapshotDiffReport getSnapshotDiffReport(String path, String fromSnapshot, String toSnapshot) throws IOException { - SnapshotDiffInfo diffs = null; + SnapshotDiffReport diffs; checkOperation(OperationCategory.READ); final FSPermissionChecker pc = getPermissionChecker(); readLock(); @@ -7497,13 +7506,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } finally { readUnlock(); } - + if (auditLog.isInfoEnabled() && isExternalInvocation()) { logAuditEvent(true, "computeSnapshotDiff", null, null, null); } - return diffs != null ? diffs.generateReport() : new SnapshotDiffReport( - path, fromSnapshot, toSnapshot, - Collections. emptyList()); + return diffs; } private void checkSubtreeReadPermission(final FSPermissionChecker pc, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index e6645da4f97..2020c521988 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.Deque; import java.util.HashMap; import java.util.Iterator; @@ -29,8 +28,6 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.server.namenode.Content; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; @@ -41,7 +38,6 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.Quota; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; import org.apache.hadoop.hdfs.util.Diff; import org.apache.hadoop.hdfs.util.Diff.Container; @@ -161,43 +157,6 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { } } } - - /** - * Interpret the diff and generate a list of {@link DiffReportEntry}. - * @param parentPath The relative path of the parent. - * @param fromEarlier True indicates {@code diff=later-earlier}, - * False indicates {@code diff=earlier-later} - * @return A list of {@link DiffReportEntry} as the diff report. - */ - public List generateReport(byte[][] parentPath, - boolean fromEarlier, Map renameMap) { - List list = new ArrayList(); - List created = getList(ListType.CREATED); - List deleted = getList(ListType.DELETED); - byte[][] fullPath = new byte[parentPath.length + 1][]; - System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length); - for (INode cnode : created) { - RenameEntry entry = renameMap.get(cnode.getId()); - if (entry == null || !entry.isRename()) { - fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); - list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE - : DiffType.DELETE, fullPath)); - } - } - for (INode dnode : deleted) { - RenameEntry entry = renameMap.get(dnode.getId()); - if (entry != null && entry.isRename()) { - list.add(new DiffReportEntry(DiffType.RENAME, - fromEarlier ? entry.getSourcePath() : entry.getTargetPath(), - fromEarlier ? entry.getTargetPath() : entry.getSourcePath())); - } else { - fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); - list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE - : DiffType.CREATE, fullPath)); - } - } - return list; - } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java index eef92157091..5a6ed75d206 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java @@ -21,22 +21,14 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.namenode.Content; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; @@ -56,7 +48,6 @@ import org.apache.hadoop.util.Time; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.primitives.SignedBytes; /** * Directories where taking snapshots is allowed. @@ -79,164 +70,6 @@ public class INodeDirectorySnapshottable extends INodeDirectory { } return (INodeDirectorySnapshottable)dir; } - - /** - * A class describing the difference between snapshots of a snapshottable - * directory. - */ - public static class SnapshotDiffInfo { - /** Compare two inodes based on their full names */ - public static final Comparator INODE_COMPARATOR = - new Comparator() { - @Override - public int compare(INode left, INode right) { - if (left == null) { - return right == null ? 0 : -1; - } else { - if (right == null) { - return 1; - } else { - int cmp = compare(left.getParent(), right.getParent()); - return cmp == 0 ? SignedBytes.lexicographicalComparator().compare( - left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp; - } - } - } - }; - - static class RenameEntry { - private byte[][] sourcePath; - private byte[][] targetPath; - - void setSource(INode source, byte[][] sourceParentPath) { - Preconditions.checkState(sourcePath == null); - sourcePath = new byte[sourceParentPath.length + 1][]; - System.arraycopy(sourceParentPath, 0, sourcePath, 0, - sourceParentPath.length); - sourcePath[sourcePath.length - 1] = source.getLocalNameBytes(); - } - - void setTarget(INode target, byte[][] targetParentPath) { - targetPath = new byte[targetParentPath.length + 1][]; - System.arraycopy(targetParentPath, 0, targetPath, 0, - targetParentPath.length); - targetPath[targetPath.length - 1] = target.getLocalNameBytes(); - } - - void setTarget(byte[][] targetPath) { - this.targetPath = targetPath; - } - - boolean isRename() { - return sourcePath != null && targetPath != null; - } - - byte[][] getSourcePath() { - return sourcePath; - } - - byte[][] getTargetPath() { - return targetPath; - } - } - - /** The root directory of the snapshots */ - private final INodeDirectorySnapshottable snapshotRoot; - /** The starting point of the difference */ - private final Snapshot from; - /** The end point of the difference */ - private final Snapshot to; - /** - * A map recording modified INodeFile and INodeDirectory and their relative - * path corresponding to the snapshot root. Sorted based on their names. - */ - private final SortedMap diffMap = - new TreeMap(INODE_COMPARATOR); - /** - * A map capturing the detailed difference about file creation/deletion. - * Each key indicates a directory whose children have been changed between - * the two snapshots, while its associated value is a {@link ChildrenDiff} - * storing the changes (creation/deletion) happened to the children (files). - */ - private final Map dirDiffMap = - new HashMap(); - - private final Map renameMap = - new HashMap(); - - SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start, - Snapshot end) { - this.snapshotRoot = snapshotRoot; - this.from = start; - this.to = end; - } - - /** Add a dir-diff pair */ - private void addDirDiff(INodeDirectory dir, byte[][] relativePath, - ChildrenDiff diff) { - dirDiffMap.put(dir, diff); - diffMap.put(dir, relativePath); - // detect rename - for (INode created : diff.getList(ListType.CREATED)) { - if (created.isReference()) { - RenameEntry entry = getEntry(created.getId()); - if (entry.getTargetPath() == null) { - entry.setTarget(created, relativePath); - } - } - } - for (INode deleted : diff.getList(ListType.DELETED)) { - if (deleted instanceof INodeReference.WithName) { - RenameEntry entry = getEntry(deleted.getId()); - entry.setSource(deleted, relativePath); - } - } - } - - private RenameEntry getEntry(long inodeId) { - RenameEntry entry = renameMap.get(inodeId); - if (entry == null) { - entry = new RenameEntry(); - renameMap.put(inodeId, entry); - } - return entry; - } - - private void setRenameTarget(long inodeId, byte[][] path) { - getEntry(inodeId).setTarget(path); - } - - /** Add a modified file */ - private void addFileDiff(INodeFile file, byte[][] relativePath) { - diffMap.put(file, relativePath); - } - - /** @return True if {@link #from} is earlier than {@link #to} */ - private boolean isFromEarlier() { - return Snapshot.ID_COMPARATOR.compare(from, to) < 0; - } - - /** - * Generate a {@link SnapshotDiffReport} based on detailed diff information. - * @return A {@link SnapshotDiffReport} describing the difference - */ - public SnapshotDiffReport generateReport() { - List diffReportList = new ArrayList(); - for (INode node : diffMap.keySet()) { - diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap - .get(node), null)); - if (node.isDirectory()) { - ChildrenDiff dirDiff = dirDiffMap.get(node); - List subList = dirDiff.generateReport( - diffMap.get(node), isFromEarlier(), renameMap); - diffReportList.addAll(subList); - } - } - return new SnapshotDiffReport(snapshotRoot.getFullPathName(), - Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), - diffReportList); - } - } /** * Snapshots of this directory in ascending order of snapshot names. @@ -496,9 +329,9 @@ public class INodeDirectorySnapshottable extends INodeDirectory { private void computeDiffRecursively(INode node, List parentPath, SnapshotDiffInfo diffReport) { final Snapshot earlierSnapshot = diffReport.isFromEarlier() ? - diffReport.from : diffReport.to; + diffReport.getFrom() : diffReport.getTo(); final Snapshot laterSnapshot = diffReport.isFromEarlier() ? - diffReport.to : diffReport.from; + diffReport.getTo() : diffReport.getFrom(); byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]); if (node.isDirectory()) { final ChildrenDiff diff = new ChildrenDiff(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java new file mode 100644 index 00000000000..f709f87daa4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff; +import org.apache.hadoop.hdfs.util.Diff.ListType; + +import com.google.common.base.Preconditions; +import com.google.common.primitives.SignedBytes; + +/** + * A class describing the difference between snapshots of a snapshottable + * directory. + */ +class SnapshotDiffInfo { + /** Compare two inodes based on their full names */ + public static final Comparator INODE_COMPARATOR = + new Comparator() { + @Override + public int compare(INode left, INode right) { + if (left == null) { + return right == null ? 0 : -1; + } else { + if (right == null) { + return 1; + } else { + int cmp = compare(left.getParent(), right.getParent()); + return cmp == 0 ? SignedBytes.lexicographicalComparator().compare( + left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp; + } + } + } + }; + + static class RenameEntry { + private byte[][] sourcePath; + private byte[][] targetPath; + + void setSource(INode source, byte[][] sourceParentPath) { + Preconditions.checkState(sourcePath == null); + sourcePath = new byte[sourceParentPath.length + 1][]; + System.arraycopy(sourceParentPath, 0, sourcePath, 0, + sourceParentPath.length); + sourcePath[sourcePath.length - 1] = source.getLocalNameBytes(); + } + + void setTarget(INode target, byte[][] targetParentPath) { + targetPath = new byte[targetParentPath.length + 1][]; + System.arraycopy(targetParentPath, 0, targetPath, 0, + targetParentPath.length); + targetPath[targetPath.length - 1] = target.getLocalNameBytes(); + } + + void setTarget(byte[][] targetPath) { + this.targetPath = targetPath; + } + + boolean isRename() { + return sourcePath != null && targetPath != null; + } + + byte[][] getSourcePath() { + return sourcePath; + } + + byte[][] getTargetPath() { + return targetPath; + } + } + + /** The root directory of the snapshots */ + private final INodeDirectorySnapshottable snapshotRoot; + /** The starting point of the difference */ + private final Snapshot from; + /** The end point of the difference */ + private final Snapshot to; + /** + * A map recording modified INodeFile and INodeDirectory and their relative + * path corresponding to the snapshot root. Sorted based on their names. + */ + private final SortedMap diffMap = + new TreeMap(INODE_COMPARATOR); + /** + * A map capturing the detailed difference about file creation/deletion. + * Each key indicates a directory whose children have been changed between + * the two snapshots, while its associated value is a {@link ChildrenDiff} + * storing the changes (creation/deletion) happened to the children (files). + */ + private final Map dirDiffMap = + new HashMap(); + + private final Map renameMap = + new HashMap(); + + SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start, + Snapshot end) { + this.snapshotRoot = snapshotRoot; + this.from = start; + this.to = end; + } + + /** Add a dir-diff pair */ + void addDirDiff(INodeDirectory dir, byte[][] relativePath, ChildrenDiff diff) { + dirDiffMap.put(dir, diff); + diffMap.put(dir, relativePath); + // detect rename + for (INode created : diff.getList(ListType.CREATED)) { + if (created.isReference()) { + RenameEntry entry = getEntry(created.getId()); + if (entry.getTargetPath() == null) { + entry.setTarget(created, relativePath); + } + } + } + for (INode deleted : diff.getList(ListType.DELETED)) { + if (deleted instanceof INodeReference.WithName) { + RenameEntry entry = getEntry(deleted.getId()); + entry.setSource(deleted, relativePath); + } + } + } + + Snapshot getFrom() { + return from; + } + + Snapshot getTo() { + return to; + } + + private RenameEntry getEntry(long inodeId) { + RenameEntry entry = renameMap.get(inodeId); + if (entry == null) { + entry = new RenameEntry(); + renameMap.put(inodeId, entry); + } + return entry; + } + + void setRenameTarget(long inodeId, byte[][] path) { + getEntry(inodeId).setTarget(path); + } + + /** Add a modified file */ + void addFileDiff(INodeFile file, byte[][] relativePath) { + diffMap.put(file, relativePath); + } + + /** @return True if {@link #from} is earlier than {@link #to} */ + boolean isFromEarlier() { + return Snapshot.ID_COMPARATOR.compare(from, to) < 0; + } + + /** + * Generate a {@link SnapshotDiffReport} based on detailed diff information. + * @return A {@link SnapshotDiffReport} describing the difference + */ + public SnapshotDiffReport generateReport() { + List diffReportList = new ArrayList(); + for (INode node : diffMap.keySet()) { + diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap + .get(node), null)); + if (node.isDirectory()) { + List subList = generateReport(dirDiffMap.get(node), + diffMap.get(node), isFromEarlier(), renameMap); + diffReportList.addAll(subList); + } + } + return new SnapshotDiffReport(snapshotRoot.getFullPathName(), + Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), + diffReportList); + } + + /** + * Interpret the ChildrenDiff and generate a list of {@link DiffReportEntry}. + * @param dirDiff The ChildrenDiff. + * @param parentPath The relative path of the parent. + * @param fromEarlier True indicates {@code diff=later-earlier}, + * False indicates {@code diff=earlier-later} + * @param renameMap A map containing information about rename operations. + * @return A list of {@link DiffReportEntry} as the diff report. + */ + private List generateReport(ChildrenDiff dirDiff, + byte[][] parentPath, boolean fromEarlier, Map renameMap) { + List list = new ArrayList(); + List created = dirDiff.getList(ListType.CREATED); + List deleted = dirDiff.getList(ListType.DELETED); + byte[][] fullPath = new byte[parentPath.length + 1][]; + System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length); + for (INode cnode : created) { + RenameEntry entry = renameMap.get(cnode.getId()); + if (entry == null || !entry.isRename()) { + fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); + list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE + : DiffType.DELETE, fullPath)); + } + } + for (INode dnode : deleted) { + RenameEntry entry = renameMap.get(dnode.getId()); + if (entry != null && entry.isRename()) { + list.add(new DiffReportEntry(DiffType.RENAME, + fromEarlier ? entry.getSourcePath() : entry.getTargetPath(), + fromEarlier ? entry.getTargetPath() : entry.getSourcePath())); + } else { + fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); + list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE + : DiffType.CREATE, fullPath)); + } + } + return list; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 4f6a6211358..ef830893210 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -30,9 +30,11 @@ import java.util.concurrent.atomic.AtomicInteger; import javax.management.ObjectName; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshotInfo; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -40,7 +42,6 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodesInPath; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo; import org.apache.hadoop.metrics2.util.MBeans; /** @@ -361,12 +362,13 @@ public class SnapshotManager implements SnapshotStatsMXBean { * Compute the difference between two snapshots of a directory, or between a * snapshot of the directory and its current tree. */ - public SnapshotDiffInfo diff(final String path, final String from, + public SnapshotDiffReport diff(final String path, final String from, final String to) throws IOException { if ((from == null || from.isEmpty()) && (to == null || to.isEmpty())) { // both fromSnapshot and toSnapshot indicate the current tree - return null; + return new SnapshotDiffReport(path, from, to, + Collections. emptyList()); } // Find the source root directory path where the snapshots were taken. @@ -374,8 +376,10 @@ public class SnapshotManager implements SnapshotStatsMXBean { INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString()); final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable .valueOf(inodesInPath.getLastINode(), path); - - return snapshotRoot.computeDiff(from, to); + + final SnapshotDiffInfo diffs = snapshotRoot.computeDiff(from, to); + return diffs != null ? diffs.generateReport() : new SnapshotDiffReport( + path, from, to, Collections. emptyList()); } public void clearSnapshottableDirs() {