From 350dcaf61679c83d0fe34824f1e2a3da80d7b70b Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Thu, 18 May 2023 15:53:26 +0800 Subject: [PATCH] HDFS-17010. Add a subtree test to TestSnapshotDiffReport. (#5656) --- .../snapshot/TestSnapRootDescendantDiff.java | 38 +++++++- .../snapshot/TestSnapshotDiffReport.java | 88 ++++++++++++++----- 2 files changed, 103 insertions(+), 23 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapRootDescendantDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapRootDescendantDiff.java index d2be71d9597..46895957213 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapRootDescendantDiff.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapRootDescendantDiff.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,10 +19,13 @@ import static org.junit.Assert.fail; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; @@ -32,7 +35,19 @@ /** * Test snapshot diff report for the snapshot root descendant directory. */ -public class TestSnapRootDescendantDiff extends TestSnapshotDiffReport { +public class TestSnapRootDescendantDiff { + { + SnapshotTestHelper.disableLogs(); + } + + private final Path dir = new Path("/" + getClass().getSimpleName()); + private final Path sub1 = new Path(dir, "sub1"); + + protected Configuration conf; + protected MiniDFSCluster cluster; + protected DistributedFileSystem hdfs; + private final Map snapshotNumberMap = new HashMap<>(); + @Before public void setUp() throws Exception { conf = new Configuration(); @@ -59,6 +74,25 @@ public void tearDown() throws Exception { } } + private Path getSnapRootDir() { + return sub1; + } + + private String genSnapshotName(Path snapshotDir) { + int sNum = -1; + if (snapshotNumberMap.containsKey(snapshotDir)) { + sNum = snapshotNumberMap.get(snapshotDir); + } + snapshotNumberMap.put(snapshotDir, ++sNum); + return "s" + sNum; + } + + void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs) + throws Exception { + TestSnapshotDiffReport.modifyAndCreateSnapshot( + modifyDir, snapshotDirs, hdfs, this::genSnapshotName); + } + @Test public void testNonSnapRootDiffReport() throws Exception { Path subsub1 = new Path(getSnapRootDir(), "subsub1"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java index 1d50e50d426..e3b85022167 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -30,6 +30,7 @@ import java.util.Random; import java.util.List; import java.util.ArrayList; +import java.util.function.Function; import org.apache.commons.collections.list.TreeList; import org.apache.hadoop.conf.Configuration; @@ -50,6 +51,8 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotException; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; +import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -58,7 +61,6 @@ import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Assert; -import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -70,6 +72,10 @@ public class TestSnapshotDiffReport { private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotDiffReport.class); + + { + SnapshotTestHelper.disableLogs(); + } private static final long SEED = 0; private static final short REPLICATION = 3; private static final short REPLICATION_1 = 2; @@ -80,9 +86,9 @@ public class TestSnapshotDiffReport { private final Path dir = new Path("/TestSnapshot"); private final Path sub1 = new Path(dir, "sub1"); - protected Configuration conf; - protected MiniDFSCluster cluster; - protected DistributedFileSystem hdfs; + private Configuration conf; + private MiniDFSCluster cluster; + private DistributedFileSystem hdfs; private final HashMap snapshotNumberMap = new HashMap(); @Before @@ -112,10 +118,6 @@ public void tearDown() throws Exception { } } - protected Path getSnapRootDir() { - return sub1; - } - private String genSnapshotName(Path snapshotDir) { int sNum = -1; if (snapshotNumberMap.containsKey(snapshotDir)) { @@ -125,11 +127,16 @@ private String genSnapshotName(Path snapshotDir) { return "s" + sNum; } + void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs) + throws Exception { + modifyAndCreateSnapshot(modifyDir, snapshotDirs, hdfs, this::genSnapshotName); + } /** * Create/modify/delete files under a given directory, also create snapshots * of directories. */ - protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs) + static void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs, + DistributedFileSystem hdfs, Function getSnapshotName) throws Exception { Path file10 = new Path(modifyDir, "file10"); Path file11 = new Path(modifyDir, "file11"); @@ -147,7 +154,7 @@ protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs) // create snapshot for (Path snapshotDir : snapshotDirs) { hdfs.allowSnapshot(snapshotDir); - hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir)); + hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir)); } // delete file11 @@ -165,7 +172,7 @@ protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs) // create snapshot for (Path snapshotDir : snapshotDirs) { - hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir)); + hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir)); } // create file11 again @@ -183,7 +190,7 @@ protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs) // create snapshot for (Path snapshotDir : snapshotDirs) { - hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir)); + hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir)); } // modify file10 hdfs.setReplication(file10, (short) (REPLICATION + 1)); @@ -317,10 +324,6 @@ public void testDiffReport() throws Exception { @Test(timeout = 60000) public void testSnapRootDescendantDiffReport() throws Exception { - Assume.assumeTrue(conf.getBoolean( - DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT, - DFSConfigKeys. - DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT_DEFAULT)); Path subSub = new Path(sub1, "subsub1"); Path subSubSub = new Path(subSub, "subsubsub1"); Path nonSnapDir = new Path(dir, "non_snap"); @@ -579,10 +582,6 @@ private void verifyDescendantDiffReports(final Path snapDir, @Test public void testSnapRootDescendantDiffReportWithRename() throws Exception { - Assume.assumeTrue(conf.getBoolean( - DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT, - DFSConfigKeys. - DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT_DEFAULT)); Path subSub = new Path(sub1, "subsub1"); Path subSubSub = new Path(subSub, "subsubsub1"); Path nonSnapDir = new Path(dir, "non_snap"); @@ -1603,4 +1602,51 @@ public void testSnapshotDiffReportRemoteIterator2() throws Exception { + "supported for snapshotDiffReport between two snapshots")); } } + + @Test + public void testSubtrees() throws Exception { + final Path root = new Path("/"); + final Path foo = new Path(root, "foo"); + final Path bar = new Path(foo, "bar"); + hdfs.mkdirs(bar); + modifyAndCreateSnapshot(bar, new Path[]{root}); + + final SnapshottableDirectoryStatus[] snapshottables + = hdfs.getSnapshottableDirListing(); + Assert.assertEquals(1, snapshottables.length); + Assert.assertEquals(3, snapshottables[0].getSnapshotNumber()); + + final SnapshotStatus[] statuses = hdfs.getSnapshotListing(root); + Assert.assertEquals(3, statuses.length); + for (int i = 0; i < statuses.length; i++) { + final SnapshotStatus s = statuses[i]; + LOG.info("Snapshot #{}: {}", s.getSnapshotID(), s.getFullPath()); + Assert.assertEquals(i, s.getSnapshotID()); + } + + for (int i = 0; i <= 2; i++) { + for (int j = 0; j <= 2; j++) { + assertDiff(root, foo, bar, "s" + i, "s" + j); + } + } + } + + private void assertDiff(Path root, Path foo, Path bar, + String from, String to) throws Exception { + final String barDiff = diff(bar, from, to); + final String fooDiff = diff(foo, from, to); + Assert.assertEquals(barDiff, fooDiff.replace("/bar", "")); + + final String rootDiff = diff(root, from, to); + Assert.assertEquals(fooDiff, rootDiff.replace("/foo", "")); + Assert.assertEquals(barDiff, rootDiff.replace("/foo/bar", "")); + } + + private String diff(Path path, String from, String to) throws Exception { + final SnapshotDiffReport diff = hdfs.getSnapshotDiffReport(path, from, to); + LOG.info("DIFF {} from {} to {}", path, from, to); + LOG.info("{}", diff); + final String report = diff.toString(); + return report.substring(report.indexOf(":") + 1); + } }