diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7e3c1f73af3..eefbb0ffcbe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -537,6 +537,9 @@ Release 2.4.0 - UNRELEASED HDFS-5979. Typo and logger fix for fsimage PB code. (wang) + HDFS-5962. Mtime and atime are not persisted for symbolic links. (Akira + Ajisaka via kihwal) + BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 5018aa03f84..57fd984a6de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -279,7 +279,8 @@ private INodeSymlink loadINodeSymlink(INodeSection.INode n) { parent.getLoaderContext().getStringTable()); INodeSymlink sym = new INodeSymlink(n.getId(), n.getName().toByteArray(), - permissions, 0, 0, s.getTarget().toStringUtf8()); + permissions, s.getModificationTime(), s.getAccessTime(), + s.getTarget().toStringUtf8()); return sym; } @@ -482,7 +483,9 @@ private void save(OutputStream out, INodeSymlink n) throws IOException { INodeSection.INodeSymlink.Builder b = INodeSection.INodeSymlink .newBuilder() .setPermission(buildPermissionStatus(n, state.getStringMap())) - .setTarget(ByteString.copyFrom(n.getSymlink())); + .setTarget(ByteString.copyFrom(n.getSymlink())) + .setModificationTime(n.getModificationTime()) + .setAccessTime(n.getAccessTime()); INodeSection.INode r = buildINodeCommon(n) .setType(INodeSection.INode.Type.SYMLINK).setSymlink(b).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsrPBImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsrPBImage.java index 7e1cea0e401..4af555ec446 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsrPBImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsrPBImage.java @@ -170,8 +170,9 @@ private void listINode(String parent, INode inode) { PermissionStatus p = FSImageFormatPBINode.Loader.loadPermission( d.getPermission(), stringTable); out.print(String.format("-%s - %8s %10s %10s %10d %s%s -> %s\n", p - .getPermission().toString(), p.getUserName(), p.getGroupName(), 0, 0, - parent, inode.getName().toStringUtf8(), d.getTarget().toStringUtf8())); + .getPermission().toString(), p.getUserName(), p.getGroupName(), d + .getModificationTime(), 0, parent, inode.getName().toStringUtf8(), + d.getTarget().toStringUtf8())); } break; default: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java index 1f43b5b3cd4..a4609103356 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -289,8 +289,9 @@ private void dumpINodeSection(InputStream in) throws IOException { } private void dumpINodeSymlink(INodeSymlink s) { - o("permission", dumpPermission(s.getPermission())).o("target", - s.getTarget().toStringUtf8()); + o("permission", dumpPermission(s.getPermission())) + .o("target", s.getTarget().toStringUtf8()) + .o("mtime", s.getModificationTime()).o("atime", s.getAccessTime()); } private void dumpNameSection(InputStream in) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto index e2f167e9e03..8dd087752fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto @@ -130,6 +130,8 @@ message INodeSection { message INodeSymlink { optional fixed64 permission = 1; optional bytes target = 2; + optional uint64 modificationTime = 3; + optional uint64 accessTime = 4; } message INode { @@ -300,4 +302,4 @@ message CacheManagerSection { required uint32 numDirectives = 3; // repeated CachePoolInfoProto pools // repeated CacheDirectiveInfoProto directives -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 552b091b7b4..338522de809 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; @@ -135,4 +136,51 @@ public void testDigest() throws IOException { } } } + + /** + * Ensure mtime and atime can be loaded from fsimage. + */ + @Test(timeout=60000) + public void testLoadMtimeAtime() throws Exception { + Configuration conf = new Configuration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + DistributedFileSystem hdfs = cluster.getFileSystem(); + String userDir = hdfs.getHomeDirectory().toUri().getPath().toString(); + Path file = new Path(userDir, "file"); + Path dir = new Path(userDir, "/dir"); + Path link = new Path(userDir, "/link"); + hdfs.createNewFile(file); + hdfs.mkdirs(dir); + hdfs.createSymlink(file, link, false); + + long mtimeFile = hdfs.getFileStatus(file).getModificationTime(); + long atimeFile = hdfs.getFileStatus(file).getAccessTime(); + long mtimeDir = hdfs.getFileStatus(dir).getModificationTime(); + long mtimeLink = hdfs.getFileLinkStatus(link).getModificationTime(); + long atimeLink = hdfs.getFileLinkStatus(link).getAccessTime(); + + // save namespace and restart cluster + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(1).build(); + cluster.waitActive(); + hdfs = cluster.getFileSystem(); + + assertEquals(mtimeFile, hdfs.getFileStatus(file).getModificationTime()); + assertEquals(atimeFile, hdfs.getFileStatus(file).getAccessTime()); + assertEquals(mtimeDir, hdfs.getFileStatus(dir).getModificationTime()); + assertEquals(mtimeLink, hdfs.getFileLinkStatus(link).getModificationTime()); + assertEquals(atimeLink, hdfs.getFileLinkStatus(link).getAccessTime()); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } }