From 1096917649fd951be633e5619518764f23cca645 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 1 Apr 2013 23:24:42 +0000 Subject: [PATCH] HDFS-4611. Update FSImage for INodeReference. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1463332 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES.HDFS-2802.txt | 2 + .../DelegationTokenSecretManager.java | 8 +- .../hdfs/server/namenode/FSDirectory.java | 4 +- .../server/namenode/FSImageCompression.java | 19 +- .../hdfs/server/namenode/FSImageFormat.java | 75 ++++--- .../server/namenode/FSImageSerialization.java | 59 ++++-- .../hdfs/server/namenode/FSNamesystem.java | 5 +- .../hadoop/hdfs/server/namenode/INode.java | 34 ++-- .../hdfs/server/namenode/INodeDirectory.java | 9 +- .../hdfs/server/namenode/INodeReference.java | 51 +++-- .../namenode/snapshot/AbstractINodeDiff.java | 16 +- .../namenode/snapshot/FileWithSnapshot.java | 9 +- .../snapshot/INodeDirectorySnapshottable.java | 5 + .../snapshot/INodeDirectoryWithSnapshot.java | 24 ++- .../server/namenode/snapshot/Snapshot.java | 17 +- .../snapshot/SnapshotFSImageFormat.java | 183 +++++++++++------- .../namenode/snapshot/SnapshotManager.java | 31 ++- .../namenode/TestFSImageWithSnapshot.java | 21 +- .../namenode/snapshot/SnapshotTestHelper.java | 17 +- .../snapshot/TestNestedSnapshots.java | 7 +- .../snapshot/TestRenameWithSnapshots.java | 107 ++++++++++ .../namenode/snapshot/TestSnapshot.java | 29 ++- 22 files changed, 515 insertions(+), 217 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt index baead25c229..6c6929fef04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt @@ -223,3 +223,5 @@ Branch-2802 Snapshot (Unreleased) HDFS-4637. INodeDirectory#replaceSelf4Quota may incorrectly convert a newly created directory to an INodeDirectoryWithSnapshot. (Jing Zhao via szetszwo) + + HDFS-4611. Update FSImage for INodeReference. (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java index d18ac9f456d..e233c816f9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.security.token.delegation; -import java.io.DataInputStream; +import java.io.DataInput; import java.io.DataOutputStream; import java.io.IOException; import java.io.InterruptedIOException; @@ -110,7 +110,7 @@ public class DelegationTokenSecretManager * @param in input stream to read fsimage * @throws IOException */ - public synchronized void loadSecretManagerState(DataInputStream in) + public synchronized void loadSecretManagerState(DataInput in) throws IOException { if (running) { // a safety check @@ -266,7 +266,7 @@ public class DelegationTokenSecretManager /** * Private helper methods to load Delegation tokens from fsimage */ - private synchronized void loadCurrentTokens(DataInputStream in) + private synchronized void loadCurrentTokens(DataInput in) throws IOException { int numberOfTokens = in.readInt(); for (int i = 0; i < numberOfTokens; i++) { @@ -282,7 +282,7 @@ public class DelegationTokenSecretManager * @param in * @throws IOException */ - private synchronized void loadAllKeys(DataInputStream in) throws IOException { + private synchronized void loadAllKeys(DataInput in) throws IOException { int numberOfKeys = in.readInt(); for (int i = 0; i < numberOfKeys; i++) { DelegationKey value = new DelegationKey(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index e77769aeca3..49993a3250c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -570,7 +570,8 @@ public class FSDirectory implements Closeable { + " because the source can not be removed"); return false; } - srcChild.setLocalName(dstComponents[dstComponents.length - 1]); + //TODO: setLocalName breaks created/deleted lists + srcChild.setLocalName(dstIIP.getLastLocalName()); // add src to the destination added = addLastINodeNoQuotaCheck(dstIIP, srcChild); @@ -750,6 +751,7 @@ public class FSDirectory implements Closeable { undoRemoveDst = true; } } + //TODO: setLocalName breaks created/deleted lists srcChild.setLocalName(dstIIP.getLastLocalName()); // add src as dst to complete rename diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageCompression.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageCompression.java index cd5092a669b..e0a46f15445 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageCompression.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageCompression.java @@ -17,23 +17,23 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.InputStream; -import java.io.OutputStream; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; -import org.apache.hadoop.io.Text; - /** * Simple container class that handles support for compressed fsimage files. */ @@ -108,15 +108,14 @@ class FSImageCompression { * underlying IO fails. */ static FSImageCompression readCompressionHeader( - Configuration conf, - DataInputStream dis) throws IOException + Configuration conf, DataInput in) throws IOException { - boolean isCompressed = dis.readBoolean(); + boolean isCompressed = in.readBoolean(); if (!isCompressed) { return createNoopCompression(); } else { - String codecClassName = Text.readString(dis); + String codecClassName = Text.readString(in); return createCompression(conf, codecClassName); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 61e0ae4e61d..74364e3746d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.util.Time.now; +import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; @@ -58,6 +59,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileUnderConstructio import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.Text; @@ -186,6 +188,9 @@ public class FSImageFormat { private long imgTxId; /** The MD5 sum of the loaded file */ private MD5Hash imgDigest; + + private Map snapshotMap = null; + private final ReferenceMap referenceMap = new ReferenceMap(); Loader(Configuration conf, FSNamesystem namesystem) { this.conf = conf; @@ -267,7 +272,7 @@ public class FSImageFormat { } if (supportSnapshot) { - namesystem.getSnapshotManager().read(in); + snapshotMap = namesystem.getSnapshotManager().read(in, this); } // read compression related info @@ -331,7 +336,7 @@ public class FSImageFormat { * * @param in Image input stream */ - private void loadLocalNameINodesWithSnapshot(DataInputStream in) + private void loadLocalNameINodesWithSnapshot(DataInput in) throws IOException { assert LayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION, getLayoutVersion()); @@ -350,7 +355,7 @@ public class FSImageFormat { * @param in image input stream * @throws IOException */ - private void loadLocalNameINodes(long numFiles, DataInputStream in) + private void loadLocalNameINodes(long numFiles, DataInput in) throws IOException { assert LayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION, getLayoutVersion()); @@ -373,20 +378,20 @@ public class FSImageFormat { /** * Load information about root, and use the information to update the root * directory of NameSystem. - * @param in The {@link DataInputStream} instance to read. + * @param in The {@link DataInput} instance to read. */ - private void loadRoot(DataInputStream in) throws IOException { + private void loadRoot(DataInput in) throws IOException { // load root if (in.readShort() != 0) { throw new IOException("First node is not root"); } - final INodeWithAdditionalFields root = loadINode(null, false, in); + final INodeDirectory root = loadINode(null, false, in).asDirectory(); // update the root's attributes updateRootAttr(root); } /** Load children nodes for the parent directory. */ - private int loadChildren(INodeDirectory parent, DataInputStream in) + private int loadChildren(INodeDirectory parent, DataInput in) throws IOException { int numChildren = in.readInt(); for (int i = 0; i < numChildren; i++) { @@ -399,9 +404,9 @@ public class FSImageFormat { /** * Load a directory when snapshot is supported. - * @param in The {@link DataInputStream} instance to read. + * @param in The {@link DataInput} instance to read. */ - private void loadDirectoryWithSnapshot(DataInputStream in) + private void loadDirectoryWithSnapshot(DataInput in) throws IOException { // Step 1. Identify the parent INode String parentPath = FSImageSerialization.readString(in); @@ -443,7 +448,7 @@ public class FSImageFormat { * @return number of child inodes read * @throws IOException */ - private int loadDirectory(DataInputStream in) throws IOException { + private int loadDirectory(DataInput in) throws IOException { String parentPath = FSImageSerialization.readString(in); final INodeDirectory parent = INodeDirectory.valueOf( namesystem.dir.rootDir.getNode(parentPath, true), parentPath); @@ -458,19 +463,19 @@ public class FSImageFormat { * @throws IOException if any error occurs */ private void loadFullNameINodes(long numFiles, - DataInputStream in) throws IOException { + DataInput in) throws IOException { byte[][] pathComponents; byte[][] parentPath = {{}}; FSDirectory fsDir = namesystem.dir; INodeDirectory parentINode = fsDir.rootDir; for (long i = 0; i < numFiles; i++) { pathComponents = FSImageSerialization.readPathComponents(in); - final INodeWithAdditionalFields newNode = loadINode( + final INode newNode = loadINode( pathComponents[pathComponents.length-1], false, in); if (isRoot(pathComponents)) { // it is the root // update the root's attributes - updateRootAttr(newNode); + updateRootAttr(newNode.asDirectory()); continue; } // check if the new inode belongs to the same parent @@ -527,12 +532,9 @@ public class FSImageFormat { } public INode loadINodeWithLocalName(boolean isSnapshotINode, - DataInputStream in) throws IOException { - final byte[] localName = new byte[in.readShort()]; - in.readFully(localName); - final INode inode = loadINode(localName, isSnapshotINode, in); - inode.setLocalName(localName); - return inode; + DataInput in) throws IOException { + final byte[] localName = FSImageSerialization.readLocalName(in); + return loadINode(localName, isSnapshotINode, in); } /** @@ -541,8 +543,8 @@ public class FSImageFormat { * @param in data input stream from which image is read * @return an inode */ - INodeWithAdditionalFields loadINode(final byte[] localName, boolean isSnapshotINode, - DataInputStream in) throws IOException { + INode loadINode(final byte[] localName, boolean isSnapshotINode, + DataInput in) throws IOException { final int imgVersion = getLayoutVersion(); final long inodeId = namesystem.allocateNewInodeId(); @@ -632,12 +634,27 @@ public class FSImageFormat { final PermissionStatus permissions = PermissionStatus.read(in); return new INodeSymlink(inodeId, localName, permissions, modificationTime, atime, symlink); + } else if (numBlocks == -3) { + //reference + + final boolean isWithName = in.readBoolean(); + + final INodeReference.WithCount withCount + = referenceMap.loadINodeReferenceWithCount(isSnapshotINode, in, this); + + if (isWithName) { + return new INodeReference.WithName(null, withCount, localName); + } else { + final INodeReference ref = new INodeReference(null, withCount); + withCount.setParentReference(ref); + return ref; + } } throw new IOException("Unknown inode type: numBlocks=" + numBlocks); } - private void loadFilesUnderConstruction(DataInputStream in, + private void loadFilesUnderConstruction(DataInput in, boolean supportSnapshot) throws IOException { FSDirectory fsDir = namesystem.dir; int size = in.readInt(); @@ -665,7 +682,7 @@ public class FSImageFormat { } } - private void loadSecretManagerState(DataInputStream in) + private void loadSecretManagerState(DataInput in) throws IOException { int imgVersion = getLayoutVersion(); @@ -713,6 +730,10 @@ public class FSImageFormat { } return result; } + + public Snapshot getSnapshot(DataInput in) throws IOException { + return snapshotMap.get(in.readInt()); + } } /** @@ -727,6 +748,7 @@ public class FSImageFormat { /** The MD5 checksum of the file that was written */ private MD5Hash savedDigest; + private final ReferenceMap referenceMap = new ReferenceMap(); static private final byte[] PATH_SEPARATOR = DFSUtil.string2Bytes(Path.SEPARATOR); @@ -792,7 +814,7 @@ public class FSImageFormat { byte[] byteStore = new byte[4*HdfsConstants.MAX_PATH_LENGTH]; ByteBuffer strbuf = ByteBuffer.wrap(byteStore); // save the root - FSImageSerialization.saveINode2Image(fsDir.rootDir, out, false); + FSImageSerialization.saveINode2Image(fsDir.rootDir, out, false, referenceMap); // save the rest of the nodes saveImage(strbuf, fsDir.rootDir, out, null); // save files under construction @@ -805,6 +827,7 @@ public class FSImageFormat { context.checkCancelled(); fout.getChannel().force(true); } finally { + referenceMap.removeAllINodeReferenceWithId(); out.close(); } @@ -830,7 +853,7 @@ public class FSImageFormat { int i = 0; for(INode child : children) { // print all children first - FSImageSerialization.saveINode2Image(child, out, false); + FSImageSerialization.saveINode2Image(child, out, false, referenceMap); if (child.isDirectory()) { dirNum++; } @@ -927,7 +950,7 @@ public class FSImageFormat { dirNum += saveChildren(children, out); // 4. Write DirectoryDiff lists, if there is any. - SnapshotFSImageFormat.saveDirectoryDiffList(current, out); + SnapshotFSImageFormat.saveDirectoryDiffList(current, out, referenceMap); // Write sub-tree of sub-directories, including possible snapshots of // deleted sub-directories diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java index 33203eeb8b0..9612bd86b93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.io.DataInputStream; +import java.io.DataInput; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.ShortWritable; import org.apache.hadoop.io.Text; @@ -87,7 +88,7 @@ public class FSImageSerialization { } private static void writeBlocks(final Block[] blocks, - final DataOutputStream out) throws IOException { + final DataOutput out) throws IOException { if (blocks == null) { out.writeInt(0); } else { @@ -102,7 +103,7 @@ public class FSImageSerialization { // from the input stream // static INodeFileUnderConstruction readINodeUnderConstruction( - DataInputStream in) throws IOException { + DataInput in) throws IOException { byte[] name = readBytes(in); short blockReplication = in.readShort(); long modificationTime = in.readLong(); @@ -164,7 +165,7 @@ public class FSImageSerialization { * @param out The {@link DataOutputStream} where the fields are written * @param writeBlock Whether to write block information */ - public static void writeINodeFile(INodeFile file, DataOutputStream out, + public static void writeINodeFile(INodeFile file, DataOutput out, boolean writeUnderConstruction) throws IOException { writeLocalName(file, out); out.writeShort(file.getFileReplication()); @@ -233,17 +234,37 @@ public class FSImageSerialization { writePermissionStatus(node, out); } + /** Serialize a {@link INodeReference} node */ + private static void writeINodeReference(INodeReference ref, DataOutput out, + boolean writeUnderConstruction, ReferenceMap referenceMap + ) throws IOException { + writeLocalName(ref, out); + out.writeShort(0); // replication + out.writeLong(0); // modification time + out.writeLong(0); // access time + out.writeLong(0); // preferred block size + out.writeInt(-3); // # of blocks + + out.writeBoolean(ref instanceof INodeReference.WithName); + + final INodeReference.WithCount withCount + = (INodeReference.WithCount)ref.getReferredINode(); + referenceMap.writeINodeReferenceWithCount(withCount, out, writeUnderConstruction); + } + /** * Save one inode's attributes to the image. */ - public static void saveINode2Image(INode node, DataOutputStream out, - boolean writeUnderConstruction) + public static void saveINode2Image(INode node, DataOutput out, + boolean writeUnderConstruction, ReferenceMap referenceMap) throws IOException { - if (node.isDirectory()) { + if (node.isReference()) { + writeINodeReference(node.asReference(), out, writeUnderConstruction, referenceMap); + } else if (node.isDirectory()) { writeINodeDirectory(node.asDirectory(), out); } else if (node.isSymlink()) { writeINodeSymlink(node.asSymlink(), out); - } else { + } else if (node.isFile()) { writeINodeFile(node.asFile(), out, writeUnderConstruction); } } @@ -252,19 +273,19 @@ public class FSImageSerialization { // code is moved into this package. This method should not be called // by other code. @SuppressWarnings("deprecation") - public static String readString(DataInputStream in) throws IOException { + public static String readString(DataInput in) throws IOException { DeprecatedUTF8 ustr = TL_DATA.get().U_STR; ustr.readFields(in); return ustr.toStringChecked(); } - static String readString_EmptyAsNull(DataInputStream in) throws IOException { + static String readString_EmptyAsNull(DataInput in) throws IOException { final String s = readString(in); return s.isEmpty()? null: s; } @SuppressWarnings("deprecation") - public static void writeString(String str, DataOutputStream out) throws IOException { + public static void writeString(String str, DataOutput out) throws IOException { DeprecatedUTF8 ustr = TL_DATA.get().U_STR; ustr.set(str); ustr.write(out); @@ -272,7 +293,7 @@ public class FSImageSerialization { /** read the long value */ - static long readLong(DataInputStream in) throws IOException { + static long readLong(DataInput in) throws IOException { LongWritable ustr = TL_DATA.get().U_LONG; ustr.readFields(in); return ustr.get(); @@ -286,7 +307,7 @@ public class FSImageSerialization { } /** read short value */ - static short readShort(DataInputStream in) throws IOException { + static short readShort(DataInput in) throws IOException { ShortWritable uShort = TL_DATA.get().U_SHORT; uShort.readFields(in); return uShort.get(); @@ -301,7 +322,7 @@ public class FSImageSerialization { // Same comments apply for this method as for readString() @SuppressWarnings("deprecation") - public static byte[] readBytes(DataInputStream in) throws IOException { + public static byte[] readBytes(DataInput in) throws IOException { DeprecatedUTF8 ustr = TL_DATA.get().U_STR; ustr.readFields(in); int len = ustr.getLength(); @@ -319,7 +340,7 @@ public class FSImageSerialization { * @throws IOException */ @SuppressWarnings("deprecation") - public static byte[][] readPathComponents(DataInputStream in) + public static byte[][] readPathComponents(DataInput in) throws IOException { DeprecatedUTF8 ustr = TL_DATA.get().U_STR; @@ -328,6 +349,12 @@ public class FSImageSerialization { ustr.getLength(), (byte) Path.SEPARATOR_CHAR); } + public static byte[] readLocalName(DataInput in) throws IOException { + byte[] createdNodeName = new byte[in.readShort()]; + in.readFully(createdNodeName); + return createdNodeName; + } + private static void writeLocalName(INode inode, DataOutput out) throws IOException { final byte[] name = inode.getLocalNameBytes(); @@ -358,7 +385,7 @@ public class FSImageSerialization { } public static Block[] readCompactBlockArray( - DataInputStream in, int logVersion) throws IOException { + DataInput in, int logVersion) throws IOException { int num = WritableUtils.readVInt(in); if (num < 0) { throw new IOException("Invalid block array length: " + num); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 350d62f8280..598f621717d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -33,8 +33,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEF import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_STANDBY_CHECKPOINTS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_STANDBY_CHECKPOINTS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DEFAULT_AUDIT_LOGGER_NAME; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT; @@ -75,6 +75,7 @@ import static org.apache.hadoop.util.Time.now; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; +import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; @@ -5353,7 +5354,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, /** * @param in load the state of secret manager from input stream */ - void loadSecretManagerState(DataInputStream in) throws IOException { + void loadSecretManagerState(DataInput in) throws IOException { dtSecretManager.loadSecretManagerState(in); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index af19d1cce7f..55cf00c341f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; import java.util.ArrayList; @@ -42,7 +43,6 @@ import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.SignedBytes; -//import org.apache.hadoop.hdfs.util.EnumCounters; /** * We keep an in-memory representation of the file/block hierarchy. @@ -420,21 +420,30 @@ public abstract class INode implements Diff.Element { } @VisibleForTesting - public String getObjectString() { + public final String getObjectString() { return getClass().getSimpleName() + "@" + Integer.toHexString(super.hashCode()); } + /** @return a string description of the parent. */ @VisibleForTesting - public String toStringWithObjectType() { - return toString() + "(" + getObjectString() + ")"; + public final String getParentString() { + final INodeReference parentRef = getParentReference(); + if (parentRef != null) { + return "parentRef=" + parentRef.getLocalName() + "->"; + } else { + final INodeDirectory parentDir = getParent(); + if (parentDir != null) { + return "parentDir=" + parentDir.getLocalName() + "/"; + } else { + return "parent=null"; + } + } } @VisibleForTesting public String toDetailString() { - final INodeDirectory p = getParent(); - return toStringWithObjectType() - + ", parent=" + (p == null? null: p.toStringWithObjectType()); + return toString() + "(" + getObjectString() + "), " + getParentString(); } /** @return the parent directory */ @@ -611,6 +620,11 @@ public abstract class INode implements Diff.Element { return out.getBuffer(); } + @VisibleForTesting + public final void dumpTreeRecursively(PrintStream out) { + dumpTreeRecursively(new PrintWriter(out, true), new StringBuilder(), null); + } + /** * Dump tree recursively. * @param prefix The prefix string that each line should print. @@ -623,10 +637,8 @@ public abstract class INode implements Diff.Element { out.print(getLocalName()); out.print(" ("); out.print(getObjectString()); - out.print("), parent="); - - final INodeDirectory p = getParent(); - out.print(p == null? null: p.getLocalName() + "/"); + out.print("), "); + out.print(getParentString()); out.print(", " + getPermissionStatus(snapshot)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index 210276ab2e6..6c8e36e82af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -227,9 +227,9 @@ public class INodeDirectory extends INodeWithAdditionalFields { INodeReference.WithCount replaceChild4Reference(INode oldChild) { Preconditions.checkArgument(!oldChild.isReference()); final INodeReference.WithCount withCount - = new INodeReference.WithCount(oldChild); - final INodeReference ref = new INodeReference(withCount); - withCount.incrementReferenceCount(); + = new INodeReference.WithCount(null, oldChild); + final INodeReference ref = new INodeReference(this, withCount); + withCount.setParentReference(ref); replaceChild(oldChild, ref); return withCount; } @@ -897,6 +897,9 @@ public class INodeDirectory extends INodeWithAdditionalFields { if (this instanceof INodeDirectoryWithQuota) { out.print(((INodeDirectoryWithQuota)this).quotaString()); } + if (this instanceof Snapshot.Root) { + out.print(", snapshotId=" + snapshot.getId()); + } out.println(); if (prefix.length() >= 2) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index cc4cebe865a..895c4a34ef5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -17,11 +17,15 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.io.PrintWriter; + import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import com.google.common.base.Preconditions; + /** * An anonymous reference to an inode. * @@ -76,11 +80,9 @@ public class INodeReference extends INode { private INode referred; - INodeReference(INode referred) { - super(referred.getParent()); + public INodeReference(INode parent, INode referred) { + super(parent); this.referred = referred; - - referred.setParentReference(this); } @@ -210,7 +212,9 @@ public class INodeReference extends INode { @Override final INode recordModification(Snapshot latest) throws QuotaExceededException { - return referred.recordModification(latest); + referred.recordModification(latest); + // reference is never replaced + return this; } @Override @@ -265,22 +269,47 @@ public class INodeReference extends INode { @Override public final void clear() { super.clear(); - referred.clear(); referred = null; } + @Override + public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, + final Snapshot snapshot) { + super.dumpTreeRecursively(out, prefix, snapshot); + if (this instanceof WithCount) { + out.print(", count=" + ((WithCount)this).getReferenceCount()); + } + out.println(); + + final StringBuilder b = new StringBuilder(); + for(int i = 0; i < prefix.length(); i++) { + b.append(' '); + } + b.append("->"); + getReferredINode().dumpTreeRecursively(out, b, snapshot); + } + /** An anonymous reference with reference count. */ public static class WithCount extends INodeReference { - private int referenceCount = 0; + private int referenceCount = 1; - WithCount(INode inode) { - super(inode); + public WithCount(INodeReference parent, INode referred) { + super(parent, referred); + Preconditions.checkArgument(!referred.isReference()); + referred.setParentReference(this); } + /** @return the reference count. */ + public int getReferenceCount() { + return referenceCount; + } + + /** Increment and then return the reference count. */ public int incrementReferenceCount() { return ++referenceCount; } + /** Decrement and then return the reference count. */ public int decrementReferenceCount() { return --referenceCount; } @@ -291,8 +320,8 @@ public class INodeReference extends INode { private final byte[] name; - public WithName(WithCount referred, byte[] name) { - super(referred); + public WithName(INodeDirectory parent, WithCount referred, byte[] name) { + super(parent, referred); this.name = name; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java index cdbf7ae8e3b..c7503cee666 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java @@ -17,13 +17,13 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import java.io.DataOutputStream; +import java.io.DataOutput; import java.io.IOException; -import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.server.namenode.INode; -import org.apache.hadoop.hdfs.server.namenode.Quota; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; +import org.apache.hadoop.hdfs.server.namenode.Quota; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; import com.google.common.base.Preconditions; @@ -144,11 +144,11 @@ abstract class AbstractINodeDiff { // write root FSImageSerialization.writeINodeDirectory(root, out); } + + static Snapshot read(DataInput in, FSImageFormat.Loader loader) throws IOException { + final int snapshotId = in.readInt(); + final INode root = loader.loadINodeWithLocalName(false, in); + return new Snapshot(snapshotId, root.asDirectory(), null); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java index 2db7b655908..f00e3a49446 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java @@ -17,48 +17,51 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import java.io.DataInputStream; -import java.io.DataOutputStream; +import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot.FileDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot.FileDiffList; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.DirectoryDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.DirectoryDiffList; -import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root; +import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff; import org.apache.hadoop.hdfs.util.Diff.ListType; import org.apache.hadoop.hdfs.util.ReadOnlyList; +import com.google.common.base.Preconditions; + /** * A helper class defining static methods for reading/writing snapshot related * information from/to FSImage. */ public class SnapshotFSImageFormat { - /** * Save snapshots and snapshot quota for a snapshottable directory. * @param current The directory that the snapshots belongs to. - * @param out The {@link DataOutputStream} to write. + * @param out The {@link DataOutput} to write. * @throws IOException */ public static void saveSnapshots(INodeDirectorySnapshottable current, - DataOutputStream out) throws IOException { + DataOutput out) throws IOException { // list of snapshots in snapshotsByNames ReadOnlyList snapshots = current.getSnapshotsByNames(); out.writeInt(snapshots.size()); - for (Snapshot ss : snapshots) { - // write the snapshot - ss.write(out); + for (Snapshot s : snapshots) { + // write the snapshot id + out.writeInt(s.getId()); } // snapshot quota out.writeInt(current.getSnapshotQuota()); @@ -67,11 +70,11 @@ public class SnapshotFSImageFormat { /** * Save SnapshotDiff list for an INodeDirectoryWithSnapshot. * @param sNode The directory that the SnapshotDiff list belongs to. - * @param out The {@link DataOutputStream} to write. + * @param out The {@link DataOutput} to write. */ private static > void saveINodeDiffs(final AbstractINodeDiffList diffs, - final DataOutputStream out) throws IOException { + final DataOutput out, ReferenceMap referenceMap) throws IOException { // Record the diffs in reversed order, so that we can find the correct // reference for INodes in the created list when loading the FSImage if (diffs == null) { @@ -81,24 +84,25 @@ public class SnapshotFSImageFormat { final int size = list.size(); out.writeInt(size); for (int i = size - 1; i >= 0; i--) { - list.get(i).write(out); + list.get(i).write(out, referenceMap); } } } public static void saveDirectoryDiffList(final INodeDirectory dir, - final DataOutputStream out) throws IOException { + final DataOutput out, final ReferenceMap referenceMap + ) throws IOException { saveINodeDiffs(dir instanceof INodeDirectoryWithSnapshot? - ((INodeDirectoryWithSnapshot)dir).getDiffs(): null, out); + ((INodeDirectoryWithSnapshot)dir).getDiffs(): null, out, referenceMap); } public static void saveFileDiffList(final INodeFile file, - final DataOutputStream out) throws IOException { + final DataOutput out) throws IOException { saveINodeDiffs(file instanceof FileWithSnapshot? - ((FileWithSnapshot)file).getDiffs(): null, out); + ((FileWithSnapshot)file).getDiffs(): null, out, null); } - public static FileDiffList loadFileDiffList(DataInputStream in, + public static FileDiffList loadFileDiffList(DataInput in, FSImageFormat.Loader loader) throws IOException { final int size = in.readInt(); if (size == -1) { @@ -115,11 +119,10 @@ public class SnapshotFSImageFormat { } } - private static FileDiff loadFileDiff(FileDiff posterior, DataInputStream in, + private static FileDiff loadFileDiff(FileDiff posterior, DataInput in, FSImageFormat.Loader loader) throws IOException { // 1. Read the full path of the Snapshot root to identify the Snapshot - Snapshot snapshot = findSnapshot(FSImageSerialization.readString(in), - loader.getFSDirectoryInLoading()); + final Snapshot snapshot = loader.getSnapshot(in); // 2. Load file size final long fileSize = in.readLong(); @@ -161,17 +164,16 @@ public class SnapshotFSImageFormat { /** * Load the created list from fsimage. * @param parent The directory that the created list belongs to. - * @param in The {@link DataInputStream} to read. + * @param in The {@link DataInput} to read. * @return The created list. */ private static List loadCreatedList(INodeDirectoryWithSnapshot parent, - DataInputStream in) throws IOException { + DataInput in) throws IOException { // read the size of the created list int createdSize = in.readInt(); List createdList = new ArrayList(createdSize); for (int i = 0; i < createdSize; i++) { - byte[] createdNodeName = new byte[in.readShort()]; - in.readFully(createdNodeName); + byte[] createdNodeName = FSImageSerialization.readLocalName(in); INode created = loadCreated(createdNodeName, parent); createdList.add(created); } @@ -184,12 +186,12 @@ public class SnapshotFSImageFormat { * @param parent The directory that the deleted list belongs to. * @param createdList The created list associated with the deleted list in * the same Diff. - * @param in The {@link DataInputStream} to read. + * @param in The {@link DataInput} to read. * @param loader The {@link Loader} instance. * @return The deleted list. */ private static List loadDeletedList(INodeDirectoryWithSnapshot parent, - List createdList, DataInputStream in, FSImageFormat.Loader loader) + List createdList, DataInput in, FSImageFormat.Loader loader) throws IOException { int deletedSize = in.readInt(); List deletedList = new ArrayList(deletedSize); @@ -208,49 +210,35 @@ public class SnapshotFSImageFormat { * Load snapshots and snapshotQuota for a Snapshottable directory. * @param snapshottableParent The snapshottable directory for loading. * @param numSnapshots The number of snapshots that the directory has. - * @param in The {@link DataInputStream} instance to read. + * @param in The {@link DataInput} instance to read. * @param loader The {@link Loader} instance that this loading procedure is * using. */ public static void loadSnapshotList( INodeDirectorySnapshottable snapshottableParent, int numSnapshots, - DataInputStream in, FSImageFormat.Loader loader) throws IOException { + DataInput in, FSImageFormat.Loader loader) throws IOException { for (int i = 0; i < numSnapshots; i++) { // read snapshots - Snapshot ss = loadSnapshot(snapshottableParent, in, loader); - snapshottableParent.addSnapshot(ss); + final Snapshot s = loader.getSnapshot(in); + s.getRoot().setParent(snapshottableParent); + snapshottableParent.addSnapshot(s); } int snapshotQuota = in.readInt(); snapshottableParent.setSnapshotQuota(snapshotQuota); } - /** - * Load a {@link Snapshot} from fsimage. - * @param parent The directory that the snapshot belongs to. - * @param in The {@link DataInputStream} instance to read. - * @param loader The {@link Loader} instance that this loading procedure is - * using. - * @return The snapshot. - */ - private static Snapshot loadSnapshot(INodeDirectorySnapshottable parent, - DataInputStream in, FSImageFormat.Loader loader) throws IOException { - int snapshotId = in.readInt(); - final INode root = loader.loadINodeWithLocalName(false, in); - return new Snapshot(snapshotId, root.asDirectory(), parent); - } - /** * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot * directory. * @param dir The snapshottable directory for loading. * @param numSnapshotDiffs The number of {@link SnapshotDiff} that the * directory has. - * @param in The {@link DataInputStream} instance to read. + * @param in The {@link DataInput} instance to read. * @param loader The {@link Loader} instance that this loading procedure is * using. */ public static void loadDirectoryDiffList(INodeDirectory dir, - DataInputStream in, FSImageFormat.Loader loader) throws IOException { + DataInput in, FSImageFormat.Loader loader) throws IOException { final int size = in.readInt(); if (size != -1) { INodeDirectoryWithSnapshot withSnapshot = (INodeDirectoryWithSnapshot)dir; @@ -261,30 +249,16 @@ public class SnapshotFSImageFormat { } } - /** - * Use the given full path to a {@link Root} directory to find the - * associated snapshot. - */ - private static Snapshot findSnapshot(String sRootFullPath, FSDirectory fsdir) - throws IOException { - // find the root - INode root = fsdir.getINode(sRootFullPath); - INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable - .valueOf(root.getParent(), root.getParent().getFullPathName()); - // find the snapshot - return snapshotRoot.getSnapshot(root.getLocalNameBytes()); - } - /** * Load the snapshotINode field of {@link SnapshotDiff}. * @param snapshot The Snapshot associated with the {@link SnapshotDiff}. - * @param in The {@link DataInputStream} to read. + * @param in The {@link DataInput} to read. * @param loader The {@link Loader} instance that this loading procedure is * using. * @return The snapshotINode. */ private static INodeDirectory loadSnapshotINodeInDirectoryDiff( - Snapshot snapshot, DataInputStream in, FSImageFormat.Loader loader) + Snapshot snapshot, DataInput in, FSImageFormat.Loader loader) throws IOException { // read the boolean indicating whether snapshotINode == Snapshot.Root boolean useRoot = in.readBoolean(); @@ -300,17 +274,16 @@ public class SnapshotFSImageFormat { /** * Load {@link DirectoryDiff} from fsimage. * @param parent The directory that the SnapshotDiff belongs to. - * @param in The {@link DataInputStream} instance to read. + * @param in The {@link DataInput} instance to read. * @param loader The {@link Loader} instance that this loading procedure is * using. * @return A {@link DirectoryDiff}. */ private static DirectoryDiff loadDirectoryDiff( - INodeDirectoryWithSnapshot parent, DataInputStream in, + INodeDirectoryWithSnapshot parent, DataInput in, FSImageFormat.Loader loader) throws IOException { // 1. Read the full path of the Snapshot root to identify the Snapshot - Snapshot snapshot = findSnapshot(FSImageSerialization.readString(in), - loader.getFSDirectoryInLoading()); + final Snapshot snapshot = loader.getSnapshot(in); // 2. Load DirectoryDiff#childrenSize int childrenSize = in.readInt(); @@ -333,4 +306,78 @@ public class SnapshotFSImageFormat { return sdiff; } + + /** A reference with a fixed id for fsimage serialization. */ + private static class INodeReferenceWithId extends INodeReference { + final long id; + + private INodeReferenceWithId(WithCount parent, INode referred, long id) { + super(parent, referred); + this.id = id; + } + + /** @return the reference id. */ + private long getReferenceId() { + return id; + } + } + + /** A reference map for fsimage serialization. */ + public static class ReferenceMap { + private final Map referenceMap + = new HashMap(); + private long referenceId = 0; + + public void writeINodeReferenceWithCount(INodeReference.WithCount withCount, + DataOutput out, boolean writeUnderConstruction) throws IOException { + final INode referred = withCount.getReferredINode(); + final boolean firstReferred = !(referred instanceof INodeReferenceWithId); + out.writeBoolean(firstReferred); + + if (firstReferred) { + FSImageSerialization.saveINode2Image(referred, out, + writeUnderConstruction, this); + final long id = ++referenceId; + referenceMap.put(id, withCount); + + final INodeReferenceWithId withId = new INodeReferenceWithId( + withCount, referred, id); + withCount.setReferredINode(withId); + referred.setParentReference(withId); + } else { + final long id = ((INodeReferenceWithId)referred).getReferenceId(); + Preconditions.checkState(referenceMap.containsKey(id)); + out.writeLong(id); + } + } + + public INodeReference.WithCount loadINodeReferenceWithCount( + boolean isSnapshotINode, DataInput in, FSImageFormat.Loader loader + ) throws IOException { + final boolean firstReferred = in.readBoolean(); + + final INodeReference.WithCount withCount; + if (firstReferred) { + final INode referred = loader.loadINodeWithLocalName(isSnapshotINode, in); + withCount = new INodeReference.WithCount(null, referred); + referenceMap.put(++referenceId, withCount); + } else { + final long id = in.readLong(); + withCount = referenceMap.get(id); + withCount.incrementReferenceCount(); + } + return withCount; + } + + public void removeAllINodeReferenceWithId() { + for(INodeReference.WithCount withCount : referenceMap.values()) { + final INodeReference ref = withCount.getReferredINode().asReference(); + final INode referred = ref.getReferredINode(); + withCount.setReferredINode(referred); + referred.setParentReference(withCount); + ref.clear(); + } + referenceMap.clear(); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 2f19148ab45..51b7cbbcbc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -21,12 +21,15 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; @@ -195,23 +198,39 @@ public class SnapshotManager implements SnapshotStats { } /** - * Write {@link #snapshotCounter}, {@link #numSnapshots}, and - * {@link #numSnapshottableDirs} to the DataOutput. + * Write {@link #snapshotCounter}, {@link #numSnapshots}, + * {@link #numSnapshottableDirs} and all snapshots to the DataOutput. */ public void write(DataOutput out) throws IOException { out.writeInt(snapshotCounter); - out.writeInt(numSnapshots.get()); out.writeInt(numSnapshottableDirs.get()); + out.writeInt(numSnapshots.get()); + + // write all snapshots. + for(INodeDirectorySnapshottable snapshottableDir : snapshottables) { + for(Snapshot s : snapshottableDir.getSnapshotsByNames()) { + s.write(out); + } + } } /** * Read values of {@link #snapshotCounter}, {@link #numSnapshots}, and - * {@link #numSnapshottableDirs} from the DataInput + * {@link #numSnapshottableDirs} and all snapshots from the DataInput */ - public void read(DataInput in) throws IOException { + public Map read(DataInput in, FSImageFormat.Loader loader + ) throws IOException { snapshotCounter = in.readInt(); - numSnapshots.set(in.readInt()); numSnapshottableDirs.set(in.readInt()); + numSnapshots.set(in.readInt()); + + // read snapshots + final Map snapshotMap = new HashMap(); + for(int i = 0; i < numSnapshots.get(); i++) { + final Snapshot s = Snapshot.read(in, loader); + snapshotMap.put(s.getId(), s); + } + return snapshotMap; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java index 5b8c63e11aa..a15b79daaf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java @@ -87,7 +87,7 @@ public class TestFSImageWithSnapshot { cluster.shutdown(); } } - + /** * Create a temp fsimage file for testing. * @param dir The directory where the fsimage file resides @@ -205,7 +205,16 @@ public class TestFSImageWithSnapshot { hdfs.setReplication(sub1file1, (short) (REPLICATION - 1)); hdfs.delete(sub1file2, true); hdfs.setOwner(sub2, "dr.who", "unknown"); - hdfs.delete(sub2file2, true); + hdfs.delete(sub2file1, true); + checkImage(s); + + hdfs.createSnapshot(dir, "s" + ++s); + Path sub1_sub2file2 = new Path(sub1, "sub2file2"); + hdfs.rename(sub2file2, sub1_sub2file2); + + hdfs.rename(sub1file1, sub2file1); + // TODO: fix case hdfs.rename(sub1file1, sub1file2); + checkImage(s); } @@ -222,8 +231,14 @@ public class TestFSImageWithSnapshot { long numSnapshotBefore = fsn.getNumSnapshots(); SnapshottableDirectoryStatus[] dirBefore = hdfs.getSnapshottableDirListing(); - // restart the cluster, and format the cluster + // shutdown the cluster cluster.shutdown(); + + // dump the fsdir tree + File fsnBetween = dumpTree2File(name + "_between"); + SnapshotTestHelper.compareDumpedTreeInFile(fsnBefore, fsnBetween); + + // restart the cluster, and format the cluster cluster = new MiniDFSCluster.Builder(conf).format(true) .numDataNodes(REPLICATION).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java index f1a20089106..d938aaac8b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java @@ -38,9 +38,11 @@ import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; @@ -269,7 +271,6 @@ public class SnapshotTestHelper { /** * Generate the path for a snapshot file. * - * @param fs FileSystem instance * @param snapshotRoot of format * {@literal /.snapshot/} * @param file path to a file @@ -279,7 +280,7 @@ public class SnapshotTestHelper { * . Null if the file is not under the directory associated with the * snapshot root. */ - static Path getSnapshotFile(FileSystem fs, Path snapshotRoot, Path file) { + static Path getSnapshotFile(Path snapshotRoot, Path file) { Path rootParent = snapshotRoot.getParent(); if (rootParent != null && rootParent.getName().equals(".snapshot")) { Path snapshotDir = rootParent.getParent(); @@ -463,11 +464,11 @@ public class SnapshotTestHelper { } } } - - static void dumpTreeRecursively(INode inode) { - if (INode.LOG.isDebugEnabled()) { - inode.dumpTreeRecursively( - new PrintWriter(System.out, true), new StringBuilder(), null); - } + + public static void dumpTree(String message, MiniDFSCluster cluster + ) throws UnresolvedLinkException { + System.out.println("XXX " + message); + cluster.getNameNode().getNamesystem().getFSDirectory().getINode("/" + ).dumpTreeRecursively(System.out); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java index 29748000c71..571527974ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.ipc.RemoteException; import org.junit.AfterClass; @@ -55,7 +54,6 @@ public class TestNestedSnapshots { private static Configuration conf = new Configuration(); private static MiniDFSCluster cluster; - private static FSNamesystem fsn; private static DistributedFileSystem hdfs; @BeforeClass @@ -63,8 +61,6 @@ public class TestNestedSnapshots { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) .build(); cluster.waitActive(); - - fsn = cluster.getNamesystem(); hdfs = cluster.getFileSystem(); } @@ -112,8 +108,7 @@ public class TestNestedSnapshots { } private static void print(String message) throws UnresolvedLinkException { - System.out.println("XXX " + message); - SnapshotTestHelper.dumpTreeRecursively(fsn.getFSDirectory().getINode("/")); + SnapshotTestHelper.dumpTree(message, cluster); } private static void assertFile(Path s1, Path s2, Path file, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java new file mode 100644 index 00000000000..3a0cbc58327 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.FSDirectory; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Testing rename with snapshots. */ +public class TestRenameWithSnapshots { + { + SnapshotTestHelper.disableLogs(); + } + + private static final long SEED = 0; + + private static final short REPL = 3; + private static final long BLOCKSIZE = 1024; + + private static Configuration conf = new Configuration(); + private static MiniDFSCluster cluster; + private static FSNamesystem fsn; + private static FSDirectory fsdir; + private static DistributedFileSystem hdfs; + + @BeforeClass + public static void setUp() throws Exception { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL).build(); + cluster.waitActive(); + + fsn = cluster.getNamesystem(); + fsdir = fsn.getFSDirectory(); + + hdfs = cluster.getFileSystem(); + } + + @AfterClass + public static void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test (timeout=300000) + public void testRenameWithSnapshot() throws Exception { + final String dirStr = "/testRenameWithSnapshot"; + final String abcStr = dirStr + "/abc"; + final Path abc = new Path(abcStr); + hdfs.mkdirs(abc, new FsPermission((short)0777)); + hdfs.allowSnapshot(abcStr); + + final Path foo = new Path(abc, "foo"); + DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, REPL, SEED); + hdfs.createSnapshot(abc, "s0"); + final INode fooINode = fsdir.getINode(foo.toString()); + + final String xyzStr = dirStr + "/xyz"; + final Path xyz = new Path(xyzStr); + hdfs.mkdirs(xyz, new FsPermission((short)0777)); + final Path bar = new Path(xyz, "bar"); + hdfs.rename(foo, bar); + + final INode fooRef = fsdir.getINode( + SnapshotTestHelper.getSnapshotPath(abc, "s0", "foo").toString()); + Assert.assertTrue(fooRef.isReference()); + Assert.assertTrue(fooRef.asReference() instanceof INodeReference.WithName); + + final INodeReference.WithCount withCount + = (INodeReference.WithCount)fooRef.asReference().getReferredINode(); + Assert.assertEquals(2, withCount.getReferenceCount()); + + final INode barRef = fsdir.getINode(bar.toString()); + Assert.assertTrue(barRef.isReference()); + + Assert.assertSame(withCount, barRef.asReference().getReferredINode()); + Assert.assertSame(fooINode, withCount.asReference().getReferredINode()); + + hdfs.delete(bar, false); + Assert.assertEquals(1, withCount.getReferenceCount()); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java index 88cdcf5fb56..a7a1fe8d0b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java @@ -76,7 +76,7 @@ public class TestSnapshot { public static final int DIRECTORY_TREE_LEVEL = 5; protected Configuration conf; - protected MiniDFSCluster cluster; + protected static MiniDFSCluster cluster; protected static FSNamesystem fsn; protected static FSDirectory fsdir; protected DistributedFileSystem hdfs; @@ -222,7 +222,7 @@ public class TestSnapshot { runTestSnapshot(); } catch(Throwable t) { SnapshotTestHelper.LOG.info("FAILED", t); - SnapshotTestHelper.dumpTreeRecursively(fsdir.getINode("/")); + SnapshotTestHelper.dumpTree("FAILED", cluster); throw t; } } @@ -471,7 +471,7 @@ public class TestSnapshot { @Override void loadSnapshots() throws Exception { for (Path snapshotRoot : snapshotList) { - Path snapshotFile = SnapshotTestHelper.getSnapshotFile(fs, + Path snapshotFile = SnapshotTestHelper.getSnapshotFile( snapshotRoot, file); if (snapshotFile != null) { if (fs.exists(snapshotFile)) { @@ -501,8 +501,7 @@ public class TestSnapshot { + "\n\nfile : " + fsdir.getINode(file.toString()).toDetailString() + "\n\nsnapshotFile: " + fsdir.getINode(snapshotFile.toString()).toDetailString(); - System.out.println(s); - SnapshotTestHelper.dumpTreeRecursively(fsdir.getINode("/")); + SnapshotTestHelper.dumpTree(s, cluster); } assertEquals(s, currentStatus.toString(), originalStatus.toString()); } @@ -582,7 +581,7 @@ public class TestSnapshot { @Override void loadSnapshots() throws Exception { for (Path snapshotRoot : snapshotList) { - Path snapshotFile = SnapshotTestHelper.getSnapshotFile(fs, + Path snapshotFile = SnapshotTestHelper.getSnapshotFile( snapshotRoot, file); if (snapshotFile != null) { long snapshotFileLen = fs.exists(snapshotFile) ? fs.getFileStatus( @@ -613,9 +612,7 @@ public class TestSnapshot { + "\noriginalSnapshotFileLen = " + originalSnapshotFileLen + "\n\nfile : " + fsdir.getINode(file.toString()).toDetailString() + "\n\nsnapshotFile: " + fsdir.getINode(snapshotFile.toString()).toDetailString(); - - System.out.println(s); - SnapshotTestHelper.dumpTreeRecursively(fsdir.getINode("/")); + SnapshotTestHelper.dumpTree(s, cluster); } assertEquals(s, originalSnapshotFileLen, currentSnapshotFileLen); // Read the snapshot file out of the boundary @@ -630,9 +627,7 @@ public class TestSnapshot { + "\n readLen = " + readLen + "\n\nfile : " + fsdir.getINode(file.toString()).toDetailString() + "\n\nsnapshotFile: " + fsdir.getINode(snapshotFile.toString()).toDetailString(); - - System.out.println(s); - SnapshotTestHelper.dumpTreeRecursively(fsdir.getINode("/")); + SnapshotTestHelper.dumpTree(s, cluster); } assertEquals(s, -1, readLen); input.close(); @@ -703,7 +698,7 @@ public class TestSnapshot { @Override void loadSnapshots() throws Exception { for (Path snapshotRoot : snapshotList) { - Path snapshotFile = SnapshotTestHelper.getSnapshotFile(fs, + Path snapshotFile = SnapshotTestHelper.getSnapshotFile( snapshotRoot, file); if (snapshotFile != null) { FileStatus status = @@ -722,7 +717,7 @@ public class TestSnapshot { @Override void checkSnapshots() throws Exception { for (Path snapshotRoot : snapshotList) { - Path snapshotFile = SnapshotTestHelper.getSnapshotFile(fs, + Path snapshotFile = SnapshotTestHelper.getSnapshotFile( snapshotRoot, file); if (snapshotFile != null) { boolean computed = fs.exists(snapshotFile); @@ -755,7 +750,7 @@ public class TestSnapshot { @Override void loadSnapshots() throws Exception { for (Path snapshotRoot : snapshotList) { - boolean existence = SnapshotTestHelper.getSnapshotFile(fs, + boolean existence = SnapshotTestHelper.getSnapshotFile( snapshotRoot, file) != null; snapshotFileExistenceMap.put(snapshotRoot, existence); } @@ -770,7 +765,7 @@ public class TestSnapshot { void checkSnapshots() throws Exception { for (Path snapshotRoot : snapshotList) { boolean currentSnapshotFileExist = SnapshotTestHelper.getSnapshotFile( - fs, snapshotRoot, file) != null; + snapshotRoot, file) != null; boolean originalSnapshotFileExist = snapshotFileExistenceMap .get(snapshotRoot); assertEquals(currentSnapshotFileExist, originalSnapshotFileExist); @@ -809,7 +804,7 @@ public class TestSnapshot { @Override void loadSnapshots() throws Exception { for (Path snapshotRoot : snapshotList) { - Path snapshotDir = SnapshotTestHelper.getSnapshotFile(fs, snapshotRoot, + Path snapshotDir = SnapshotTestHelper.getSnapshotFile(snapshotRoot, changedPath); if (snapshotDir != null) { FileStatus status = fs.exists(snapshotDir) ? fs