HDFS-10998. Add unit tests for HDFS command 'dfsadmin -fetchImage' in HA. Contributed by Xiaobing Zhou

(cherry picked from commit d7d87deece)

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java

@@ -17,10 +17,15 @@
  */
 package org.apache.hadoop.hdfs;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
 import static org.junit.Assert.assertEquals;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -29,10 +34,16 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.hdfs.tools.DFSAdmin;
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.PathUtils;
 import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class TestFetchImage {
@@ -42,46 +53,89 @@ public class TestFetchImage {
   // Shamelessly stolen from NNStorage.
   private static final Pattern IMAGE_REGEX = Pattern.compile("fsimage_(\\d+)");
 
+  private MiniDFSCluster cluster;
+  private NameNode nn0 = null;
+  private NameNode nn1 = null;
+  private Configuration conf = null;
+
+  @BeforeClass
+  public static void setupImageDir() {
+    FETCHED_IMAGE_FILE.mkdirs();
+  }
+
   @AfterClass
   public static void cleanup() {
     FileUtil.fullyDelete(FETCHED_IMAGE_FILE);
   }
 
+  @Before
+  public void setupCluster() throws IOException, URISyntaxException {
+    conf = new Configuration();
+    conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 1);
+    conf.setInt(DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+    conf.setLong(DFS_BLOCK_SIZE_KEY, 1024);
+
+    cluster = new MiniDFSCluster.Builder(conf)
+        .nnTopology(MiniDFSNNTopology.simpleHATopology())
+        .numDataNodes(1)
+        .build();
+    nn0 = cluster.getNameNode(0);
+    nn1 = cluster.getNameNode(1);
+    HATestUtil.configureFailoverFs(cluster, conf);
+    cluster.waitActive();
+  }
+
   /**
    * Download a few fsimages using `hdfs dfsadmin -fetchImage ...' and verify
    * the results.
    */
-  @Test
+  @Test(timeout=30000)
-  public void testFetchImage() throws Exception {
+  public void testFetchImageHA() throws Exception {
-    FETCHED_IMAGE_FILE.mkdirs();
+    final Path parent = new Path(
-    Configuration conf = new Configuration();
+        PathUtils.getTestPath(getClass()),
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+        GenericTestUtils.getMethodName());
-    FileSystem fs = null;
-    try {
-      DFSAdmin dfsAdmin = new DFSAdmin();
-      dfsAdmin.setConf(conf);
 
+    int nnIndex = 0;
+    /* run on nn0 as active */
+    cluster.transitionToActive(nnIndex);
+    testFetchImageInternal(
+        nnIndex,
+        new Path(parent, "dir1"),
+        new Path(parent, "dir2"));
+
+    /* run on nn1 as active */
+    nnIndex = 1;
+    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
+    cluster.transitionToActive(nnIndex);
+    testFetchImageInternal(
+        nnIndex,
+        new Path(parent, "dir3"),
+        new Path(parent, "dir4"));
+  }
+
+  private void testFetchImageInternal(
+      final int nnIndex,
+      final Path dir1,
+      final Path dir2) throws Exception {
+    final Configuration dfsConf = cluster.getConfiguration(nnIndex);
+    final DFSAdmin dfsAdmin = new DFSAdmin();
+    dfsAdmin.setConf(dfsConf);
+
+    try (FileSystem fs = cluster.getFileSystem(nnIndex)) {
       runFetchImage(dfsAdmin, cluster);
       
-      fs = cluster.getFileSystem();
+      fs.mkdirs(dir1);
-      fs.mkdirs(new Path("/foo"));
+      fs.mkdirs(dir2);
-      fs.mkdirs(new Path("/foo2"));
-      fs.mkdirs(new Path("/foo3"));
       
-      cluster.getNameNodeRpc()
+      cluster.getNameNodeRpc(nnIndex).setSafeMode(
-          .setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
+          SafeModeAction.SAFEMODE_ENTER,
-      cluster.getNameNodeRpc().saveNamespace();
+          false);
-      cluster.getNameNodeRpc()
+      cluster.getNameNodeRpc(nnIndex).saveNamespace();
-          .setSafeMode(SafeModeAction.SAFEMODE_LEAVE, false);
+      cluster.getNameNodeRpc(nnIndex).setSafeMode(
+          SafeModeAction.SAFEMODE_LEAVE,
+          false);
 
       runFetchImage(dfsAdmin, cluster);
-    } finally {
-      if (fs != null) {
-        fs.close();
-      }
-      if (cluster != null) {
-        cluster.shutdown();
-      }
     }
   }