HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable. Contributed by Jing Zhao.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605169 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Haohui Mai
parent
1f9a0fd927
commit
bdf3de9f6b
|
|
@ -473,6 +473,9 @@ Release 2.5.0 - UNRELEASED
|
|||
|
||||
HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu)
|
||||
|
||||
HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable.
|
||||
(Jing Zhao via wheat9)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
|
||||
|
|
|
|||
|
|
@ -23,16 +23,14 @@ import java.util.List;
|
|||
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.DFSUtil;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
|
||||
|
||||
import com.google.common.base.Objects;
|
||||
|
||||
/**
|
||||
* This class represents to end users the difference between two snapshots of
|
||||
* the same directory, or the difference between a snapshot of the directory and
|
||||
* its current state. Instead of capturing all the details of the diff, which
|
||||
* is stored in {@link SnapshotDiffInfo}, this class only lists where the
|
||||
* changes happened and their types.
|
||||
* its current state. Instead of capturing all the details of the diff, this
|
||||
* class only lists where the changes happened and their types.
|
||||
*/
|
||||
public class SnapshotDiffReport {
|
||||
private final static String LINE_SEPARATOR = System.getProperty(
|
||||
|
|
|
|||
|
|
@ -85,7 +85,18 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
|
|||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
|
||||
import static org.apache.hadoop.util.Time.now;
|
||||
|
||||
import java.io.*;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.DataInput;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
|
|
@ -170,7 +181,6 @@ import org.apache.hadoop.hdfs.protocol.RollingUpgradeException;
|
|||
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
|
||||
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
|
||||
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
|
||||
|
|
@ -208,7 +218,6 @@ import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
|
|||
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
|
||||
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
|
||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
|
||||
|
|
@ -7484,7 +7493,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
|||
*/
|
||||
SnapshotDiffReport getSnapshotDiffReport(String path,
|
||||
String fromSnapshot, String toSnapshot) throws IOException {
|
||||
SnapshotDiffInfo diffs = null;
|
||||
SnapshotDiffReport diffs;
|
||||
checkOperation(OperationCategory.READ);
|
||||
final FSPermissionChecker pc = getPermissionChecker();
|
||||
readLock();
|
||||
|
|
@ -7498,13 +7507,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
|||
} finally {
|
||||
readUnlock();
|
||||
}
|
||||
|
||||
|
||||
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
|
||||
logAuditEvent(true, "computeSnapshotDiff", null, null, null);
|
||||
}
|
||||
return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
|
||||
path, fromSnapshot, toSnapshot,
|
||||
Collections.<DiffReportEntry> emptyList());
|
||||
return diffs;
|
||||
}
|
||||
|
||||
private void checkSubtreeReadPermission(final FSPermissionChecker pc,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
|
|||
import java.io.DataOutput;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Deque;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
|
|
@ -29,8 +28,6 @@ import java.util.Map;
|
|||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
|
||||
import org.apache.hadoop.hdfs.server.namenode.Content;
|
||||
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
|
||||
|
|
@ -41,7 +38,6 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
|
|||
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
|
||||
import org.apache.hadoop.hdfs.server.namenode.Quota;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
|
||||
import org.apache.hadoop.hdfs.util.Diff;
|
||||
import org.apache.hadoop.hdfs.util.Diff.Container;
|
||||
|
|
@ -161,43 +157,6 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpret the diff and generate a list of {@link DiffReportEntry}.
|
||||
* @param parentPath The relative path of the parent.
|
||||
* @param fromEarlier True indicates {@code diff=later-earlier},
|
||||
* False indicates {@code diff=earlier-later}
|
||||
* @return A list of {@link DiffReportEntry} as the diff report.
|
||||
*/
|
||||
public List<DiffReportEntry> generateReport(byte[][] parentPath,
|
||||
boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
|
||||
List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
|
||||
List<INode> created = getList(ListType.CREATED);
|
||||
List<INode> deleted = getList(ListType.DELETED);
|
||||
byte[][] fullPath = new byte[parentPath.length + 1][];
|
||||
System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
|
||||
for (INode cnode : created) {
|
||||
RenameEntry entry = renameMap.get(cnode.getId());
|
||||
if (entry == null || !entry.isRename()) {
|
||||
fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
|
||||
list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
|
||||
: DiffType.DELETE, fullPath));
|
||||
}
|
||||
}
|
||||
for (INode dnode : deleted) {
|
||||
RenameEntry entry = renameMap.get(dnode.getId());
|
||||
if (entry != null && entry.isRename()) {
|
||||
list.add(new DiffReportEntry(DiffType.RENAME,
|
||||
fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
|
||||
fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
|
||||
} else {
|
||||
fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
|
||||
list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
|
||||
: DiffType.CREATE, fullPath));
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -21,22 +21,14 @@ import java.io.IOException;
|
|||
import java.io.PrintWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hdfs.DFSUtil;
|
||||
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotException;
|
||||
import org.apache.hadoop.hdfs.server.namenode.Content;
|
||||
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
|
||||
|
|
@ -56,7 +48,6 @@ import org.apache.hadoop.util.Time;
|
|||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.primitives.SignedBytes;
|
||||
|
||||
/**
|
||||
* Directories where taking snapshots is allowed.
|
||||
|
|
@ -79,164 +70,6 @@ public class INodeDirectorySnapshottable extends INodeDirectory {
|
|||
}
|
||||
return (INodeDirectorySnapshottable)dir;
|
||||
}
|
||||
|
||||
/**
|
||||
* A class describing the difference between snapshots of a snapshottable
|
||||
* directory.
|
||||
*/
|
||||
public static class SnapshotDiffInfo {
|
||||
/** Compare two inodes based on their full names */
|
||||
public static final Comparator<INode> INODE_COMPARATOR =
|
||||
new Comparator<INode>() {
|
||||
@Override
|
||||
public int compare(INode left, INode right) {
|
||||
if (left == null) {
|
||||
return right == null ? 0 : -1;
|
||||
} else {
|
||||
if (right == null) {
|
||||
return 1;
|
||||
} else {
|
||||
int cmp = compare(left.getParent(), right.getParent());
|
||||
return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
|
||||
left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static class RenameEntry {
|
||||
private byte[][] sourcePath;
|
||||
private byte[][] targetPath;
|
||||
|
||||
void setSource(INode source, byte[][] sourceParentPath) {
|
||||
Preconditions.checkState(sourcePath == null);
|
||||
sourcePath = new byte[sourceParentPath.length + 1][];
|
||||
System.arraycopy(sourceParentPath, 0, sourcePath, 0,
|
||||
sourceParentPath.length);
|
||||
sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
|
||||
}
|
||||
|
||||
void setTarget(INode target, byte[][] targetParentPath) {
|
||||
targetPath = new byte[targetParentPath.length + 1][];
|
||||
System.arraycopy(targetParentPath, 0, targetPath, 0,
|
||||
targetParentPath.length);
|
||||
targetPath[targetPath.length - 1] = target.getLocalNameBytes();
|
||||
}
|
||||
|
||||
void setTarget(byte[][] targetPath) {
|
||||
this.targetPath = targetPath;
|
||||
}
|
||||
|
||||
boolean isRename() {
|
||||
return sourcePath != null && targetPath != null;
|
||||
}
|
||||
|
||||
byte[][] getSourcePath() {
|
||||
return sourcePath;
|
||||
}
|
||||
|
||||
byte[][] getTargetPath() {
|
||||
return targetPath;
|
||||
}
|
||||
}
|
||||
|
||||
/** The root directory of the snapshots */
|
||||
private final INodeDirectorySnapshottable snapshotRoot;
|
||||
/** The starting point of the difference */
|
||||
private final Snapshot from;
|
||||
/** The end point of the difference */
|
||||
private final Snapshot to;
|
||||
/**
|
||||
* A map recording modified INodeFile and INodeDirectory and their relative
|
||||
* path corresponding to the snapshot root. Sorted based on their names.
|
||||
*/
|
||||
private final SortedMap<INode, byte[][]> diffMap =
|
||||
new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
|
||||
/**
|
||||
* A map capturing the detailed difference about file creation/deletion.
|
||||
* Each key indicates a directory whose children have been changed between
|
||||
* the two snapshots, while its associated value is a {@link ChildrenDiff}
|
||||
* storing the changes (creation/deletion) happened to the children (files).
|
||||
*/
|
||||
private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
|
||||
new HashMap<INodeDirectory, ChildrenDiff>();
|
||||
|
||||
private final Map<Long, RenameEntry> renameMap =
|
||||
new HashMap<Long, RenameEntry>();
|
||||
|
||||
SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
|
||||
Snapshot end) {
|
||||
this.snapshotRoot = snapshotRoot;
|
||||
this.from = start;
|
||||
this.to = end;
|
||||
}
|
||||
|
||||
/** Add a dir-diff pair */
|
||||
private void addDirDiff(INodeDirectory dir, byte[][] relativePath,
|
||||
ChildrenDiff diff) {
|
||||
dirDiffMap.put(dir, diff);
|
||||
diffMap.put(dir, relativePath);
|
||||
// detect rename
|
||||
for (INode created : diff.getList(ListType.CREATED)) {
|
||||
if (created.isReference()) {
|
||||
RenameEntry entry = getEntry(created.getId());
|
||||
if (entry.getTargetPath() == null) {
|
||||
entry.setTarget(created, relativePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (INode deleted : diff.getList(ListType.DELETED)) {
|
||||
if (deleted instanceof INodeReference.WithName) {
|
||||
RenameEntry entry = getEntry(deleted.getId());
|
||||
entry.setSource(deleted, relativePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private RenameEntry getEntry(long inodeId) {
|
||||
RenameEntry entry = renameMap.get(inodeId);
|
||||
if (entry == null) {
|
||||
entry = new RenameEntry();
|
||||
renameMap.put(inodeId, entry);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
private void setRenameTarget(long inodeId, byte[][] path) {
|
||||
getEntry(inodeId).setTarget(path);
|
||||
}
|
||||
|
||||
/** Add a modified file */
|
||||
private void addFileDiff(INodeFile file, byte[][] relativePath) {
|
||||
diffMap.put(file, relativePath);
|
||||
}
|
||||
|
||||
/** @return True if {@link #from} is earlier than {@link #to} */
|
||||
private boolean isFromEarlier() {
|
||||
return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a {@link SnapshotDiffReport} based on detailed diff information.
|
||||
* @return A {@link SnapshotDiffReport} describing the difference
|
||||
*/
|
||||
public SnapshotDiffReport generateReport() {
|
||||
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
|
||||
for (INode node : diffMap.keySet()) {
|
||||
diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
|
||||
.get(node), null));
|
||||
if (node.isDirectory()) {
|
||||
ChildrenDiff dirDiff = dirDiffMap.get(node);
|
||||
List<DiffReportEntry> subList = dirDiff.generateReport(
|
||||
diffMap.get(node), isFromEarlier(), renameMap);
|
||||
diffReportList.addAll(subList);
|
||||
}
|
||||
}
|
||||
return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
|
||||
Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
|
||||
diffReportList);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Snapshots of this directory in ascending order of snapshot names.
|
||||
|
|
@ -496,9 +329,9 @@ public class INodeDirectorySnapshottable extends INodeDirectory {
|
|||
private void computeDiffRecursively(INode node, List<byte[]> parentPath,
|
||||
SnapshotDiffInfo diffReport) {
|
||||
final Snapshot earlierSnapshot = diffReport.isFromEarlier() ?
|
||||
diffReport.from : diffReport.to;
|
||||
diffReport.getFrom() : diffReport.getTo();
|
||||
final Snapshot laterSnapshot = diffReport.isFromEarlier() ?
|
||||
diffReport.to : diffReport.from;
|
||||
diffReport.getTo() : diffReport.getFrom();
|
||||
byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
|
||||
if (node.isDirectory()) {
|
||||
final ChildrenDiff diff = new ChildrenDiff();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,242 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hdfs.server.namenode.snapshot;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INode;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
|
||||
import org.apache.hadoop.hdfs.util.Diff.ListType;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.primitives.SignedBytes;
|
||||
|
||||
/**
|
||||
* A class describing the difference between snapshots of a snapshottable
|
||||
* directory.
|
||||
*/
|
||||
class SnapshotDiffInfo {
|
||||
/** Compare two inodes based on their full names */
|
||||
public static final Comparator<INode> INODE_COMPARATOR =
|
||||
new Comparator<INode>() {
|
||||
@Override
|
||||
public int compare(INode left, INode right) {
|
||||
if (left == null) {
|
||||
return right == null ? 0 : -1;
|
||||
} else {
|
||||
if (right == null) {
|
||||
return 1;
|
||||
} else {
|
||||
int cmp = compare(left.getParent(), right.getParent());
|
||||
return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
|
||||
left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static class RenameEntry {
|
||||
private byte[][] sourcePath;
|
||||
private byte[][] targetPath;
|
||||
|
||||
void setSource(INode source, byte[][] sourceParentPath) {
|
||||
Preconditions.checkState(sourcePath == null);
|
||||
sourcePath = new byte[sourceParentPath.length + 1][];
|
||||
System.arraycopy(sourceParentPath, 0, sourcePath, 0,
|
||||
sourceParentPath.length);
|
||||
sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
|
||||
}
|
||||
|
||||
void setTarget(INode target, byte[][] targetParentPath) {
|
||||
targetPath = new byte[targetParentPath.length + 1][];
|
||||
System.arraycopy(targetParentPath, 0, targetPath, 0,
|
||||
targetParentPath.length);
|
||||
targetPath[targetPath.length - 1] = target.getLocalNameBytes();
|
||||
}
|
||||
|
||||
void setTarget(byte[][] targetPath) {
|
||||
this.targetPath = targetPath;
|
||||
}
|
||||
|
||||
boolean isRename() {
|
||||
return sourcePath != null && targetPath != null;
|
||||
}
|
||||
|
||||
byte[][] getSourcePath() {
|
||||
return sourcePath;
|
||||
}
|
||||
|
||||
byte[][] getTargetPath() {
|
||||
return targetPath;
|
||||
}
|
||||
}
|
||||
|
||||
/** The root directory of the snapshots */
|
||||
private final INodeDirectorySnapshottable snapshotRoot;
|
||||
/** The starting point of the difference */
|
||||
private final Snapshot from;
|
||||
/** The end point of the difference */
|
||||
private final Snapshot to;
|
||||
/**
|
||||
* A map recording modified INodeFile and INodeDirectory and their relative
|
||||
* path corresponding to the snapshot root. Sorted based on their names.
|
||||
*/
|
||||
private final SortedMap<INode, byte[][]> diffMap =
|
||||
new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
|
||||
/**
|
||||
* A map capturing the detailed difference about file creation/deletion.
|
||||
* Each key indicates a directory whose children have been changed between
|
||||
* the two snapshots, while its associated value is a {@link ChildrenDiff}
|
||||
* storing the changes (creation/deletion) happened to the children (files).
|
||||
*/
|
||||
private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
|
||||
new HashMap<INodeDirectory, ChildrenDiff>();
|
||||
|
||||
private final Map<Long, RenameEntry> renameMap =
|
||||
new HashMap<Long, RenameEntry>();
|
||||
|
||||
SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
|
||||
Snapshot end) {
|
||||
this.snapshotRoot = snapshotRoot;
|
||||
this.from = start;
|
||||
this.to = end;
|
||||
}
|
||||
|
||||
/** Add a dir-diff pair */
|
||||
void addDirDiff(INodeDirectory dir, byte[][] relativePath, ChildrenDiff diff) {
|
||||
dirDiffMap.put(dir, diff);
|
||||
diffMap.put(dir, relativePath);
|
||||
// detect rename
|
||||
for (INode created : diff.getList(ListType.CREATED)) {
|
||||
if (created.isReference()) {
|
||||
RenameEntry entry = getEntry(created.getId());
|
||||
if (entry.getTargetPath() == null) {
|
||||
entry.setTarget(created, relativePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (INode deleted : diff.getList(ListType.DELETED)) {
|
||||
if (deleted instanceof INodeReference.WithName) {
|
||||
RenameEntry entry = getEntry(deleted.getId());
|
||||
entry.setSource(deleted, relativePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Snapshot getFrom() {
|
||||
return from;
|
||||
}
|
||||
|
||||
Snapshot getTo() {
|
||||
return to;
|
||||
}
|
||||
|
||||
private RenameEntry getEntry(long inodeId) {
|
||||
RenameEntry entry = renameMap.get(inodeId);
|
||||
if (entry == null) {
|
||||
entry = new RenameEntry();
|
||||
renameMap.put(inodeId, entry);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
void setRenameTarget(long inodeId, byte[][] path) {
|
||||
getEntry(inodeId).setTarget(path);
|
||||
}
|
||||
|
||||
/** Add a modified file */
|
||||
void addFileDiff(INodeFile file, byte[][] relativePath) {
|
||||
diffMap.put(file, relativePath);
|
||||
}
|
||||
|
||||
/** @return True if {@link #from} is earlier than {@link #to} */
|
||||
boolean isFromEarlier() {
|
||||
return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a {@link SnapshotDiffReport} based on detailed diff information.
|
||||
* @return A {@link SnapshotDiffReport} describing the difference
|
||||
*/
|
||||
public SnapshotDiffReport generateReport() {
|
||||
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
|
||||
for (INode node : diffMap.keySet()) {
|
||||
diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
|
||||
.get(node), null));
|
||||
if (node.isDirectory()) {
|
||||
List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node),
|
||||
diffMap.get(node), isFromEarlier(), renameMap);
|
||||
diffReportList.addAll(subList);
|
||||
}
|
||||
}
|
||||
return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
|
||||
Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
|
||||
diffReportList);
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpret the ChildrenDiff and generate a list of {@link DiffReportEntry}.
|
||||
* @param dirDiff The ChildrenDiff.
|
||||
* @param parentPath The relative path of the parent.
|
||||
* @param fromEarlier True indicates {@code diff=later-earlier},
|
||||
* False indicates {@code diff=earlier-later}
|
||||
* @param renameMap A map containing information about rename operations.
|
||||
* @return A list of {@link DiffReportEntry} as the diff report.
|
||||
*/
|
||||
private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff,
|
||||
byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
|
||||
List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
|
||||
List<INode> created = dirDiff.getList(ListType.CREATED);
|
||||
List<INode> deleted = dirDiff.getList(ListType.DELETED);
|
||||
byte[][] fullPath = new byte[parentPath.length + 1][];
|
||||
System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
|
||||
for (INode cnode : created) {
|
||||
RenameEntry entry = renameMap.get(cnode.getId());
|
||||
if (entry == null || !entry.isRename()) {
|
||||
fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
|
||||
list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
|
||||
: DiffType.DELETE, fullPath));
|
||||
}
|
||||
}
|
||||
for (INode dnode : deleted) {
|
||||
RenameEntry entry = renameMap.get(dnode.getId());
|
||||
if (entry != null && entry.isRename()) {
|
||||
list.add(new DiffReportEntry(DiffType.RENAME,
|
||||
fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
|
||||
fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
|
||||
} else {
|
||||
fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
|
||||
list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
|
||||
: DiffType.CREATE, fullPath));
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
}
|
||||
|
|
@ -30,9 +30,11 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
import javax.management.ObjectName;
|
||||
|
||||
import org.apache.hadoop.hdfs.DFSUtil;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotException;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotInfo;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
|
||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
|
||||
|
|
@ -40,7 +42,6 @@ import org.apache.hadoop.hdfs.server.namenode.INode;
|
|||
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
|
||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
|
||||
import org.apache.hadoop.metrics2.util.MBeans;
|
||||
|
||||
/**
|
||||
|
|
@ -361,12 +362,13 @@ public class SnapshotManager implements SnapshotStatsMXBean {
|
|||
* Compute the difference between two snapshots of a directory, or between a
|
||||
* snapshot of the directory and its current tree.
|
||||
*/
|
||||
public SnapshotDiffInfo diff(final String path, final String from,
|
||||
public SnapshotDiffReport diff(final String path, final String from,
|
||||
final String to) throws IOException {
|
||||
if ((from == null || from.isEmpty())
|
||||
&& (to == null || to.isEmpty())) {
|
||||
// both fromSnapshot and toSnapshot indicate the current tree
|
||||
return null;
|
||||
return new SnapshotDiffReport(path, from, to,
|
||||
Collections.<DiffReportEntry> emptyList());
|
||||
}
|
||||
|
||||
// Find the source root directory path where the snapshots were taken.
|
||||
|
|
@ -374,8 +376,10 @@ public class SnapshotManager implements SnapshotStatsMXBean {
|
|||
INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString());
|
||||
final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable
|
||||
.valueOf(inodesInPath.getLastINode(), path);
|
||||
|
||||
return snapshotRoot.computeDiff(from, to);
|
||||
|
||||
final SnapshotDiffInfo diffs = snapshotRoot.computeDiff(from, to);
|
||||
return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
|
||||
path, from, to, Collections.<DiffReportEntry> emptyList());
|
||||
}
|
||||
|
||||
public void clearSnapshottableDirs() {
|
||||
|
|
|
|||
Loading…
Reference in New Issue