From 12fe3afcd3ea224909bd125f5c6a7410bf2601b0 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Thu, 26 Feb 2015 10:14:40 +0800 Subject: [PATCH] HDFS-7843. A truncated file is corrupted after rollback from a rolling upgrade. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/namenode/FSNamesystem.java | 3 ++ .../hadoop/hdfs/TestRollingUpgrade.java | 52 +++++++++++++------ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6644e590f81..83517abb53f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -729,6 +729,9 @@ Release 2.7.0 - UNRELEASED HDFS-7763. fix zkfc hung issue due to not catching exception in a corner case. (Liang Xie via wang) + HDFS-7843. A truncated file is corrupted after rollback from a rolling + upgrade. (szetszwo) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index cc932a3c6e1..c7d9350167e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2131,6 +2131,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, if(!isUpgradeFinalized()) { return true; } + if (isRollingUpgrade()) { + return true; + } return file.isBlockInLatestSnapshot(blk); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java index aa3f362bb06..0dde90940f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java @@ -23,9 +23,11 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; @@ -36,6 +38,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode; +import org.apache.hadoop.hdfs.server.namenode.TestFileTruncate; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.io.IOUtils; import org.junit.Assert; @@ -260,42 +263,50 @@ public class TestRollingUpgrade { final Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); final Path foo = new Path("/foo"); final Path bar = new Path("/bar"); cluster.getFileSystem().mkdirs(foo); - startRollingUpgrade(foo, bar, cluster); - cluster.getFileSystem().rollEdits(); - cluster.getFileSystem().rollEdits(); - rollbackRollingUpgrade(foo, bar, cluster); + final Path file = new Path(foo, "file"); + final byte[] data = new byte[1024]; + DFSUtil.getRandom().nextBytes(data); + final FSDataOutputStream out = cluster.getFileSystem().create(file); + out.write(data, 0, data.length); + out.close(); - startRollingUpgrade(foo, bar, cluster); + startRollingUpgrade(foo, bar, file, data, cluster); cluster.getFileSystem().rollEdits(); cluster.getFileSystem().rollEdits(); - rollbackRollingUpgrade(foo, bar, cluster); + rollbackRollingUpgrade(foo, bar, file, data, cluster); - startRollingUpgrade(foo, bar, cluster); + startRollingUpgrade(foo, bar, file, data, cluster); + cluster.getFileSystem().rollEdits(); + cluster.getFileSystem().rollEdits(); + rollbackRollingUpgrade(foo, bar, file, data, cluster); + + startRollingUpgrade(foo, bar, file, data, cluster); cluster.restartNameNode(); - rollbackRollingUpgrade(foo, bar, cluster); + rollbackRollingUpgrade(foo, bar, file, data, cluster); - startRollingUpgrade(foo, bar, cluster); + startRollingUpgrade(foo, bar, file, data, cluster); cluster.restartNameNode(); - rollbackRollingUpgrade(foo, bar, cluster); + rollbackRollingUpgrade(foo, bar, file, data, cluster); - startRollingUpgrade(foo, bar, cluster); - rollbackRollingUpgrade(foo, bar, cluster); + startRollingUpgrade(foo, bar, file, data, cluster); + rollbackRollingUpgrade(foo, bar, file, data, cluster); - startRollingUpgrade(foo, bar, cluster); - rollbackRollingUpgrade(foo, bar, cluster); + startRollingUpgrade(foo, bar, file, data, cluster); + rollbackRollingUpgrade(foo, bar, file, data, cluster); } finally { if(cluster != null) cluster.shutdown(); } } private static void startRollingUpgrade(Path foo, Path bar, + Path file, byte[] data, MiniDFSCluster cluster) throws IOException { final DistributedFileSystem dfs = cluster.getFileSystem(); @@ -305,18 +316,27 @@ public class TestRollingUpgrade { dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); dfs.mkdirs(bar); - Assert.assertTrue(dfs.exists(foo)); Assert.assertTrue(dfs.exists(bar)); + + //truncate a file + final int newLength = DFSUtil.getRandom().nextInt(data.length - 1) + 1; + dfs.truncate(file, newLength); + TestFileTruncate.checkBlockRecovery(file, dfs); + AppendTestUtil.checkFullFile(dfs, file, newLength, data); } private static void rollbackRollingUpgrade(Path foo, Path bar, + Path file, byte[] data, MiniDFSCluster cluster) throws IOException { + final DataNodeProperties dnprop = cluster.stopDataNode(0); cluster.restartNameNode("-rollingUpgrade", "rollback"); + cluster.restartDataNode(dnprop, true); final DistributedFileSystem dfs = cluster.getFileSystem(); Assert.assertTrue(dfs.exists(foo)); Assert.assertFalse(dfs.exists(bar)); + AppendTestUtil.checkFullFile(dfs, file, data.length, data); } @Test