From dd7776b2fe158abbe0626743612adca4ad08f581 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 26 Jun 2015 11:30:59 -0700 Subject: [PATCH] HDFS-8656. Preserve compatibility of ClientProtocol#rollingUpgrade after finalization. (cherry picked from commit 60b858bfa65e0feb665e1a84784a3d45e9091c66) --- .../hadoop/hdfs/protocol/ClientProtocol.java | 3 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSNamesystem.java | 8 ++- .../hdfs/server/namenode/NameNodeMXBean.java | 5 +- .../hadoop/hdfs/TestRollingUpgrade.java | 66 +++++++++++++++---- .../server/namenode/TestNameNodeMXBean.java | 5 +- 6 files changed, 70 insertions(+), 20 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 599eb06c0bd..aa94b5bd46b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -848,7 +848,8 @@ public interface ClientProtocol { /** * Rolling upgrade operations. * @param action either query, prepare or finalize. - * @return rolling upgrade information. + * @return rolling upgrade information. On query, if no upgrade is in + * progress, returns null. */ @Idempotent public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 91088e144f2..c3911d08e03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -780,6 +780,9 @@ Release 2.7.1 - UNRELEASED HDFS-8626. Reserved RBW space is not released if creation of RBW File fails. (kanaka kumar avvaru via Arpit Agarwal) + HDFS08656. Preserve compatibility of ClientProtocol#rollingUpgrade after + finalization. (wang) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index c23d1324b3d..5d04333bfd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6752,10 +6752,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, checkOperation(OperationCategory.READ); readLock(); try { - if (rollingUpgradeInfo != null) { - boolean hasRollbackImage = this.getFSImage().hasRollbackFSImage(); - rollingUpgradeInfo.setCreatedRollbackImages(hasRollbackImage); + if (!isRollingUpgrade()) { + return null; } + Preconditions.checkNotNull(rollingUpgradeInfo); + boolean hasRollbackImage = this.getFSImage().hasRollbackFSImage(); + rollingUpgradeInfo.setCreatedRollbackImages(hasRollbackImage); return rollingUpgradeInfo; } finally { readUnlock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java index 3c317fda7d9..0e4d44594d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java @@ -81,9 +81,10 @@ public interface NameNodeMXBean { public boolean isUpgradeFinalized(); /** - * Gets the RollingUpgrade information + * Gets the RollingUpgrade information. * - * @return Rolling upgrade information + * @return Rolling upgrade information if an upgrade is in progress. Else + * (e.g. if there is no upgrade or the upgrade is finalized), returns null. */ public RollingUpgradeInfo.Bean getRollingUpgradeStatus(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java index 10c2847a6c9..9241851aa47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java @@ -19,8 +19,18 @@ package org.apache.hadoop.hdfs; import java.io.File; import java.io.IOException; +import java.lang.management.ManagementFactory; import java.util.concurrent.ThreadLocalRandom; +import javax.management.AttributeNotFoundException; +import javax.management.InstanceNotFoundException; +import javax.management.MBeanException; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.management.ReflectionException; +import javax.management.openmbean.CompositeDataSupport; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -45,6 +55,9 @@ import org.apache.hadoop.io.IOUtils; import org.junit.Assert; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; /** * This class tests rolling upgrade. @@ -55,7 +68,7 @@ public class TestRollingUpgrade { public static void runCmd(DFSAdmin dfsadmin, boolean success, String... args) throws Exception { if (success) { - Assert.assertEquals(0, dfsadmin.run(args)); + assertEquals(0, dfsadmin.run(args)); } else { Assert.assertTrue(dfsadmin.run(args) != 0); } @@ -85,6 +98,7 @@ public class TestRollingUpgrade { //illegal argument "abc" to rollingUpgrade option runCmd(dfsadmin, false, "-rollingUpgrade", "abc"); + checkMxBeanIsNull(); //query rolling upgrade runCmd(dfsadmin, true, "-rollingUpgrade"); @@ -95,11 +109,16 @@ public class TestRollingUpgrade { //query rolling upgrade runCmd(dfsadmin, true, "-rollingUpgrade", "query"); + checkMxBean(); dfs.mkdirs(bar); //finalize rolling upgrade runCmd(dfsadmin, true, "-rollingUpgrade", "finalize"); + // RollingUpgradeInfo should be null after finalization, both via + // Java API and in JMX + assertNull(dfs.rollingUpgrade(RollingUpgradeAction.QUERY)); + checkMxBeanIsNull(); dfs.mkdirs(baz); @@ -196,8 +215,8 @@ public class TestRollingUpgrade { LOG.info("START\n" + info1); //query rolling upgrade - Assert.assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY)); - + assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY)); + dfs.mkdirs(bar); cluster.shutdown(); } @@ -217,13 +236,13 @@ public class TestRollingUpgrade { Assert.assertFalse(dfs2.exists(baz)); //query rolling upgrade in cluster2 - Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); + assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); dfs2.mkdirs(baz); LOG.info("RESTART cluster 2"); cluster2.restartNameNode(); - Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); + assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); Assert.assertTrue(dfs2.exists(foo)); Assert.assertTrue(dfs2.exists(bar)); Assert.assertTrue(dfs2.exists(baz)); @@ -237,7 +256,7 @@ public class TestRollingUpgrade { LOG.info("RESTART cluster 2 again"); cluster2.restartNameNode(); - Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); + assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); Assert.assertTrue(dfs2.exists(foo)); Assert.assertTrue(dfs2.exists(bar)); Assert.assertTrue(dfs2.exists(baz)); @@ -258,9 +277,31 @@ public class TestRollingUpgrade { } } + private static CompositeDataSupport getBean() + throws MalformedObjectNameException, MBeanException, + AttributeNotFoundException, InstanceNotFoundException, + ReflectionException { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanName = + new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo"); + return (CompositeDataSupport)mbs.getAttribute(mxbeanName, + "RollingUpgradeStatus"); + } + + private static void checkMxBeanIsNull() throws Exception { + CompositeDataSupport ruBean = getBean(); + assertNull(ruBean); + } + + private static void checkMxBean() throws Exception { + CompositeDataSupport ruBean = getBean(); + assertNotEquals(0l, ruBean.get("startTime")); + assertEquals(0l, ruBean.get("finalizeTime")); + } + @Test - public void testRollback() throws IOException { - // start a cluster + public void testRollback() throws Exception { + // start a cluster final Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; try { @@ -278,10 +319,13 @@ public class TestRollingUpgrade { out.write(data, 0, data.length); out.close(); + checkMxBeanIsNull(); startRollingUpgrade(foo, bar, file, data, cluster); + checkMxBean(); cluster.getFileSystem().rollEdits(); cluster.getFileSystem().rollEdits(); rollbackRollingUpgrade(foo, bar, file, data, cluster); + checkMxBeanIsNull(); startRollingUpgrade(foo, bar, file, data, cluster); cluster.getFileSystem().rollEdits(); @@ -355,18 +399,18 @@ public class TestRollingUpgrade { // check the datanode final String dnAddr = dn.getDatanodeId().getIpcAddr(false); final String[] args1 = {"-getDatanodeInfo", dnAddr}; - Assert.assertEquals(0, dfsadmin.run(args1)); + runCmd(dfsadmin, true, args1); // issue shutdown to the datanode. final String[] args2 = {"-shutdownDatanode", dnAddr, "upgrade" }; - Assert.assertEquals(0, dfsadmin.run(args2)); + runCmd(dfsadmin, true, args2); // the datanode should be down. Thread.sleep(2000); Assert.assertFalse("DataNode should exit", dn.isDatanodeUp()); // ping should fail. - Assert.assertEquals(-1, dfsadmin.run(args1)); + assertEquals(-1, dfsadmin.run(args1)); } finally { if (cluster != null) cluster.shutdown(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 681e8a4ed55..2087c444953 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.top.TopConf; -import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator; import org.apache.hadoop.util.VersionInfo; @@ -46,8 +45,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op; -import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.TopWindow; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -197,6 +194,8 @@ public class TestNameNodeMXBean { assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() * cluster.getDataNodes().size(), mbs.getAttribute(mxbeanName, "CacheCapacity")); + assertNull("RollingUpgradeInfo should be null when there is no rolling" + + " upgrade", mbs.getAttribute(mxbeanName, "RollingUpgradeStatus")); } finally { if (cluster != null) { for (URI dir : cluster.getNameDirs(0)) {