From 2f3ceb8c65d951d57bf2bedbabafb8d63f1aa625 Mon Sep 17 00:00:00 2001
From: Akira Ajisaka <aajisaka@apache.org>
Date: Wed, 2 Mar 2016 14:13:43 +0900
Subject: [PATCH] HDFS-9766. TestDataNodeMetrics#testDataNodeTimeSpend fails
 intermittently. Contributed by Xiao Chen.

(cherry picked from commit 81fbc40f084eab85750cc7d4cc967d43d70d8333)
(cherry picked from commit e8401417269bc3515918566bb2860e3b70d1d3ed)
---
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt   |  3 ++
 .../server/datanode/TestDataNodeMetrics.java  | 48 +++++++++++--------
 2 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 677321730f1..af9ad82eb66 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -126,6 +126,9 @@ Release 2.7.3 - UNRELEASED
 
     HDFS-9880. TestDatanodeRegistration fails occasionally (kihwal)
 
+    HDFS-9766. TestDataNodeMetrics#testDataNodeTimeSpend fails intermittently.
+    (Xiao Chen via aajisaka)
+
 Release 2.7.2 - 2016-01-25
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index 5d27fe67e8a..3423f0209bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -28,9 +28,10 @@ import static org.junit.Assert.assertTrue;
 import java.io.Closeable;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.List;
-import java.util.Map;
 
+import com.google.common.base.Supplier;
 import com.google.common.collect.Lists;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Time;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -255,37 +257,43 @@ public class TestDataNodeMetrics {
    * and reading causes totalReadTime to move.
    * @throws Exception
    */
-  @Test
+  @Test(timeout=60000)
   public void testDataNodeTimeSpend() throws Exception {
     Configuration conf = new HdfsConfiguration();
     SimulatedFSDataset.setFactory(conf);
     MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
     try {
-      FileSystem fs = cluster.getFileSystem();
+      final FileSystem fs = cluster.getFileSystem();
       List<DataNode> datanodes = cluster.getDataNodes();
       assertEquals(datanodes.size(), 1);
-      DataNode datanode = datanodes.get(0);
+      final DataNode datanode = datanodes.get(0);
       MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
       final long LONG_FILE_LEN = 1024 * 1024 * 10;
 
-      long startWriteValue = getLongCounter("TotalWriteTime", rb);
-      long startReadValue = getLongCounter("TotalReadTime", rb);
+      final long startWriteValue = getLongCounter("TotalWriteTime", rb);
+      final long startReadValue = getLongCounter("TotalReadTime", rb);
+      final AtomicInteger x = new AtomicInteger(0);
 
-      for (int x =0; x < 50; x++) {
-        DFSTestUtil.createFile(fs, new Path("/time.txt."+ x),
+      // Lets Metric system update latest metrics
+      GenericTestUtils.waitFor(new Supplier<Boolean>() {
+        @Override
+        public Boolean get() {
+          x.getAndIncrement();
+          try {
+            DFSTestUtil.createFile(fs, new Path("/time.txt." + x.get()),
                 LONG_FILE_LEN, (short) 1, Time.monotonicNow());
-      }
-
-      for (int x =0; x < 50; x++) {
-        String s = DFSTestUtil.readFile(fs, new Path("/time.txt." + x));
-      }
-
-      MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
-      long endWriteValue = getLongCounter("TotalWriteTime", rbNew);
-      long endReadValue = getLongCounter("TotalReadTime", rbNew);
-
-      assertTrue(endReadValue > startReadValue);
-      assertTrue(endWriteValue > startWriteValue);
+            DFSTestUtil.readFile(fs, new Path("/time.txt." + x.get()));
+          } catch (IOException ioe) {
+            LOG.error("Caught IOException while ingesting DN metrics", ioe);
+            return false;
+          }
+          MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
+          final long endWriteValue = getLongCounter("TotalWriteTime", rbNew);
+          final long endReadValue = getLongCounter("TotalReadTime", rbNew);
+          return endWriteValue > startWriteValue
+              && endReadValue > startReadValue;
+        }
+      }, 30, 30000);
     } finally {
       if (cluster != null) {
         cluster.shutdown();