From 2d03209a06df361e4d224795ed4dce8f9414d70f Mon Sep 17 00:00:00 2001 From: bshashikant Date: Mon, 31 Aug 2020 11:29:48 +0530 Subject: [PATCH] HDFS-15542. Add identified snapshot corruption tests for ordered snapshot deletion (#2251) --- ...estFSImageWithOrderedSnapshotDeletion.java | 495 ++++++++++++++++++ 1 file changed, 495 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFSImageWithOrderedSnapshotDeletion.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFSImageWithOrderedSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFSImageWithOrderedSnapshotDeletion.java new file mode 100644 index 00000000000..dec28e4b247 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFSImageWithOrderedSnapshotDeletion.java @@ -0,0 +1,495 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.*; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespacePrintVisitor; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.event.Level; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; + +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; +import static org.junit.Assert.assertTrue; + +/** + * Test FSImage correctness with ordered snapshot deletion. + */ +public class TestFSImageWithOrderedSnapshotDeletion { + { + SnapshotTestHelper.disableLogs(); + GenericTestUtils.setLogLevel(INode.LOG, Level.TRACE); + } + + static final long SEED = 0; + static final short NUM_DATANODES = 1; + static final int BLOCKSIZE = 1024; + + private final Path dir = new Path("/TestSnapshot"); + private static final String TEST_DIR = + GenericTestUtils.getTestDir().getAbsolutePath(); + + Configuration conf; + MiniDFSCluster cluster; + FSNamesystem fsn; + DistributedFileSystem hdfs; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + conf.setBoolean(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, true); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES) + .build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + void rename(Path src, Path dst) throws Exception { + printTree("Before rename " + src + " -> " + dst); + hdfs.rename(src, dst); + printTree("After rename " + src + " -> " + dst); + } + + void createFile(Path directory, String filename) throws Exception { + final Path f = new Path(directory, filename); + DFSTestUtil.createFile(hdfs, f, 0, NUM_DATANODES, SEED); + } + + void appendFile(Path directory, String filename) throws Exception { + final Path f = new Path(directory, filename); + DFSTestUtil.appendFile(hdfs, f, "more data"); + printTree("appended " + f); + } + + void deleteSnapshot(Path directory, String snapshotName) throws Exception { + hdfs.deleteSnapshot(directory, snapshotName); + printTree("deleted snapshot " + snapshotName); + } + + @Test (timeout=60000) + public void testDoubleRename() throws Exception { + final Path parent = new Path("/parent"); + hdfs.mkdirs(parent); + final Path sub1 = new Path(parent, "sub1"); + final Path sub1foo = new Path(sub1, "foo"); + hdfs.mkdirs(sub1); + hdfs.mkdirs(sub1foo); + createFile(sub1foo, "file0"); + + printTree("before s0"); + hdfs.allowSnapshot(parent); + hdfs.createSnapshot(parent, "s0"); + + createFile(sub1foo, "file1"); + createFile(sub1foo, "file2"); + + final Path sub2 = new Path(parent, "sub2"); + hdfs.mkdirs(sub2); + final Path sub2foo = new Path(sub2, "foo"); + // mv /parent/sub1/foo to /parent/sub2/foo + rename(sub1foo, sub2foo); + + hdfs.createSnapshot(parent, "s1"); + hdfs.createSnapshot(parent, "s2"); + printTree("created snapshots: s1, s2"); + + appendFile(sub2foo, "file1"); + createFile(sub2foo, "file3"); + + final Path sub3 = new Path(parent, "sub3"); + hdfs.mkdirs(sub3); + // mv /parent/sub2/foo to /parent/sub3/foo + rename(sub2foo, sub3); + + hdfs.delete(sub3, true); + printTree("deleted " + sub3); + + deleteSnapshot(parent, "s1"); + restartCluster(); + + deleteSnapshot(parent, "s2"); + restartCluster(); + } + + private File getDumpTreeFile(String directory, String suffix) { + return new File(directory, String.format("dumpTree_%s", suffix)); + } + /** + * Dump the fsdir tree to a temp file. + * @param fileSuffix suffix of the temp file for dumping + * @return the temp file + */ + private File dumpTree2File(String fileSuffix) throws IOException { + File file = getDumpTreeFile(TEST_DIR, fileSuffix); + SnapshotTestHelper.dumpTree2File(fsn.getFSDirectory(), file); + return file; + } + + void restartCluster() throws Exception { + final File before = dumpTree2File("before.txt"); + + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + final File after = dumpTree2File("after.txt"); + SnapshotTestHelper.compareDumpedTreeInFile(before, after, true); + } + + private final PrintWriter output = new PrintWriter(System.out, true); + private int printTreeCount = 0; + + String printTree(String label) throws Exception { + output.println(); + output.println(); + output.println("***** " + printTreeCount++ + ": " + label); + final String b = + fsn.getFSDirectory().getINode("/").dumpTreeRecursively().toString(); + output.println(b); + + final String s = NamespacePrintVisitor.print2Sting(fsn); + Assert.assertEquals(b, s); + return b; + } + + @Test (timeout=60000) + public void testFSImageWithDoubleRename() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dira, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirb); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + Path file1 = new Path(dirb, "file1"); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED); + Path rennamePath = new Path(dirx, "dirb"); + // mv /dir1/dira/dirb to /dir1/dirx/dirb + hdfs.rename(dirb, rennamePath); + hdfs.createSnapshot(dir1, "s1"); + DFSTestUtil.appendFile(hdfs, new Path("/dir1/dirx/dirb/file1"), + "more data"); + Path renamePath1 = new Path(dir2, "dira"); + hdfs.mkdirs(renamePath1); + //mv dirx/dirb to /dir2/dira/dirb + hdfs.rename(rennamePath, renamePath1); + hdfs.delete(renamePath1, true); + hdfs.deleteSnapshot(dir1, "s1"); + // save namespace and restart cluster + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + + @Test (timeout=60000) + public void testFSImageWithRename1() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dirx, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + hdfs.mkdirs(dirb); + hdfs.createSnapshot(dir1, "s1"); + Path rennamePath = new Path(dira, "dirb"); + // mv /dir1/dirx/dirb to /dir1/dira/dirb + hdfs.rename(dirb, rennamePath); + hdfs.createSnapshot(dir1, "s2"); + Path diry = new Path("/dir1/dira/dirb/diry"); + hdfs.mkdirs(diry); + hdfs.createSnapshot(dir1, "s3"); + Path file1 = new Path("/dir1/dira/dirb/diry/file1"); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED); + hdfs.createSnapshot(dir1, "s4"); + hdfs.delete(new Path("/dir1/dira/dirb"), true); + hdfs.deleteSnapshot(dir1, "s1"); + hdfs.deleteSnapshot(dir1, "s3"); + // file1 should exist in the last snapshot + assertTrue(hdfs.exists( + new Path("/dir1/.snapshot/s4/dira/dirb/diry/file1"))); + + // save namespace and restart cluster + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @Test (timeout=60000) + public void testFSImageWithRename2() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dirx, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + hdfs.mkdirs(dirb); + hdfs.createSnapshot(dir1, "s1"); + Path rennamePath = new Path(dira, "dirb"); + // mv /dir1/dirx/dirb to /dir1/dira/dirb + hdfs.rename(dirb, rennamePath); + hdfs.createSnapshot(dir1, "s2"); + Path file1 = new Path("/dir1/dira/dirb/file1"); + DFSTestUtil.createFile(hdfs, + new Path( + "/dir1/dira/dirb/file1"), BLOCKSIZE, (short) 1, SEED); + hdfs.createSnapshot(dir1, "s3"); + hdfs.deleteSnapshot(dir1, "s1"); + hdfs.deleteSnapshot(dir1, "s3"); + assertTrue(hdfs.exists(file1)); + + // save namespace and restart cluster + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @Test(timeout = 60000) + public void testFSImageWithRename3() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dirx, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + hdfs.mkdirs(dirb); + hdfs.createSnapshot(dir1, "s1"); + Path rennamePath = new Path(dira, "dirb"); + // mv /dir1/dirx/dirb to /dir1/dira/dirb + hdfs.rename(dirb, rennamePath); + hdfs.createSnapshot(dir1, "s2"); + Path diry = new Path("/dir1/dira/dirb/diry"); + hdfs.mkdirs(diry); + hdfs.createSnapshot(dir1, "s3"); + Path file1 = new Path("/dir1/dira/dirb/diry/file1"); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED); + hdfs.createSnapshot(dir1, "s4"); + hdfs.delete(new Path("/dir1/dira/dirb"), true); + hdfs.deleteSnapshot(dir1, "s1"); + hdfs.deleteSnapshot(dir1, "s3"); + // file1 should exist in the last snapshot + assertTrue(hdfs.exists(new Path( + "/dir1/.snapshot/s4/dira/dirb/diry/file1"))); + + // save namespace and restart cluster + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @Test (timeout=60000) + public void testFSImageWithRename4() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dirx, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + hdfs.mkdirs(dirb); + hdfs.createSnapshot(dir1, "s1"); + Path renamePath = new Path(dira, "dirb"); + // mv /dir1/dirx/dirb to /dir1/dira/dirb + hdfs.rename(dirb, renamePath); + hdfs.createSnapshot(dir1, "s2"); + Path diry = new Path("/dir1/dira/dirb/diry"); + hdfs.mkdirs(diry); + hdfs.createSnapshot(dir1, "s3"); + Path file1 = new Path("/dir1/dira/dirb/diry/file1"); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED); + hdfs.createSnapshot(dir1, "s4"); + hdfs.delete(new Path("/dir1/dira/dirb/diry/file1"), false); + hdfs.deleteSnapshot(dir1, "s1"); + hdfs.deleteSnapshot(dir1, "s3"); + // file1 should exist in the last snapshot + assertTrue(hdfs.exists( + new Path("/dir1/.snapshot/s4/dira/dirb/diry/file1"))); + + // save namespace and restart cluster + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @Test + public void testFSImageWithRename5() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dira, "dirb"); + Path dirc = new Path(dirb, "dirc"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirb); + hdfs.mkdirs(dirc); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + Path dird = new Path(dirc, "dird"); + Path dire = new Path(dird, "dire"); + Path file1 = new Path(dire, "file1"); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED); + Path rennamePath = new Path(dirx, "dirb"); + // mv /dir1/dira/dirb to /dir1/dirx/dirb + hdfs.rename(dirb, rennamePath); + hdfs.createSnapshot(dir1, "s1"); + DFSTestUtil.appendFile(hdfs, + new Path("/dir1/dirx/dirb/dirc/dird/dire/file1"), "more data"); + Path renamePath1 = new Path(dir2, "dira"); + hdfs.mkdirs(renamePath1); + //mv dirx/dirb to /dir2/dira/dirb + hdfs.rename(rennamePath, renamePath1); + hdfs.delete(renamePath1, true); + hdfs.deleteSnapshot(dir1, "s1"); + // save namespace and restart cluster + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @Test (timeout=60000) + public void testDoubleRenamesWithSnapshotDelete() throws Exception { + final Path sub1 = new Path(dir, "sub1"); + hdfs.mkdirs(sub1); + hdfs.allowSnapshot(sub1); + final Path dir1 = new Path(sub1, "dir1"); + final Path dir2 = new Path(sub1, "dir2"); + final Path dir3 = new Path(sub1, "dir3"); + final String snap3 = "snap3"; + final String snap4 = "snap4"; + final String snap5 = "snap5"; + final String snap6 = "snap6"; + final Path foo = new Path(dir2, "foo"); + final Path bar = new Path(dir2, "bar"); + hdfs.createSnapshot(sub1, "snap1"); + hdfs.mkdirs(dir1, new FsPermission((short) 0777)); + rename(dir1, dir2); + hdfs.createSnapshot(sub1, "snap2"); + DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, (short) 1, SEED); + DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, (short) 1, SEED); + hdfs.createSnapshot(sub1, snap3); + hdfs.delete(foo, false); + DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, (short) 1, SEED); + hdfs.createSnapshot(sub1, snap4); + hdfs.delete(foo, false); + DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, (short) 1, SEED); + hdfs.createSnapshot(sub1, snap5); + rename(dir2, dir3); + hdfs.createSnapshot(sub1, snap6); + hdfs.delete(dir3, true); + deleteSnapshot(sub1, snap6); + deleteSnapshot(sub1, snap3); + // save namespace and restart Namenode + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + cluster.restartNameNode(true); + } +}