diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6f5b939db0e..f5b57cb1c2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -75,6 +75,9 @@ Release 2.0.3-alpha - Unreleased HDFS-3999. HttpFS OPEN operation expects len parameter, it should be length. (tucu) + HDFS-4006. TestCheckpoint#testSecondaryHasVeryOutOfDateImage + occasionally fails due to unexpected exit. (todd via eli) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 11ccaef31cc..4fdae270a94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -76,6 +76,7 @@ import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; /********************************************************** @@ -120,6 +121,8 @@ public class SecondaryNameNode implements Runnable { private CheckpointConf checkpointConf; private FSNamesystem namesystem; + private Thread checkpointThread; + @Override public String toString() { @@ -275,6 +278,15 @@ private void initialize(final Configuration conf, */ public void shutdown() { shouldRun = false; + if (checkpointThread != null) { + checkpointThread.interrupt(); + try { + checkpointThread.join(10000); + } catch (InterruptedException e) { + LOG.info("Interrupted waiting to join on checkpointer thread"); + Thread.currentThread().interrupt(); // maintain status + } + } try { if (infoServer != null) infoServer.stop(); } catch (Exception e) { @@ -588,12 +600,20 @@ public static void main(String[] argv) throws Exception { terminate(ret); } - // Create a never ending deamon - Daemon checkpointThread = new Daemon(secondary); - checkpointThread.start(); + secondary.startCheckpointThread(); } + public void startCheckpointThread() { + Preconditions.checkState(checkpointThread == null, + "Should not already have a thread"); + Preconditions.checkState(shouldRun, "shouldRun should be true"); + + checkpointThread = new Daemon(this); + checkpointThread.start(); + } + + /** * Container for parsed command-line options. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index ce6865df391..3323b20ba90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -30,6 +30,8 @@ import java.io.File; import java.io.IOException; import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; @@ -74,6 +76,7 @@ import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatcher; @@ -127,7 +130,22 @@ static void writeFile(FileSystem fileSys, Path name, int repl) stm.close(); } - + @After + public void checkForSNNThreads() { + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + + ThreadInfo[] infos = threadBean.getThreadInfo(threadBean.getAllThreadIds(), 20); + for (ThreadInfo info : infos) { + if (info == null) continue; + LOG.info("Check thread: " + info.getThreadName()); + if (info.getThreadName().contains("SecondaryNameNode")) { + fail("Leaked thread: " + info + "\n" + + Joiner.on("\n").join(info.getStackTrace())); + } + } + LOG.info("--------"); + } + static void checkFile(FileSystem fileSys, Path name, int repl) throws IOException { assertTrue(fileSys.exists(name)); @@ -1744,7 +1762,7 @@ public void testCheckpointWithSeparateDirsAfterNameFails() throws Exception { /** * Test that the 2NN triggers a checkpoint after the configurable interval */ - @Test + @Test(timeout=30000) public void testCheckpointTriggerOnTxnCount() throws Exception { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1758,8 +1776,7 @@ public void testCheckpointTriggerOnTxnCount() throws Exception { .format(true).build(); FileSystem fs = cluster.getFileSystem(); secondary = startSecondaryNameNode(conf); - Thread t = new Thread(secondary); - t.start(); + secondary.startCheckpointThread(); final NNStorage storage = secondary.getFSImage().getStorage(); // 2NN should checkpoint at startup