HDDS-87:Fix test failures with uninitialized storageLocation field in storageReport Contributed by Shashikant Banerjee

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java

@@ -16,10 +16,13 @@
  */
 package org.apache.hadoop.hdds.scm;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.hdds.protocol
     .proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
 import org.apache.hadoop.hdds.protocol.proto
         .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.StorageTypeProto;
 import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 
@@ -58,19 +61,35 @@ public static DatanodeDetails getDatanodeDetails(SCMNodeManager nodeManager,
    * Create Node Report object.
    * @return SCMNodeReport
    */
-  public static SCMNodeReport createNodeReport() {
+  public static SCMNodeReport createNodeReport(List<SCMStorageReport> reports) {
     SCMNodeReport.Builder nodeReport = SCMNodeReport.newBuilder();
-    for (int i = 0; i < 1; i++) {
+    nodeReport.addAllStorageReport(reports);
-      SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
-      nodeReport.addStorageReport(i, srb.setStorageUuid("disk")
-          .setCapacity(100)
-          .setScmUsed(10)
-          .setRemaining(90)
-          .build());
-    }
     return nodeReport.build();
   }
 
+  /**
+   * Create SCM Storage Report object.
+   * @return list of SCMStorageReport
+   */
+  public static List<SCMStorageReport> createStorageReport(long capacity,
+      long used, long remaining, String path, StorageTypeProto type, String id,
+      int count) {
+    List<SCMStorageReport> reportList = new ArrayList<>();
+    for (int i = 0; i < count; i++) {
+      Preconditions.checkNotNull(path);
+      Preconditions.checkNotNull(id);
+      SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+      srb.setStorageUuid(id).setStorageLocation(path).setCapacity(capacity)
+          .setScmUsed(used).setRemaining(remaining);
+      StorageTypeProto storageTypeProto =
+          type == null ? StorageTypeProto.DISK : type;
+      srb.setStorageType(storageTypeProto);
+      reportList.add(srb.build());
+    }
+    return reportList;
+  }
+
+
   /**
    * Get specified number of DatanodeDetails and registered them with node
    * manager.

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java

@@ -35,8 +35,6 @@
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -140,13 +138,12 @@ public void testContainerPlacementCapacity() throws IOException,
         TestUtils.getListOfRegisteredDatanodeDetails(nodeManager, nodeCount);
     try {
       for (DatanodeDetails datanodeDetails : datanodes) {
-        SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
+        String id = UUID.randomUUID().toString();
-        SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+        String path = testDir.getAbsolutePath() + "/" + id;
-        srb.setStorageUuid(UUID.randomUUID().toString());
+        List<SCMStorageReport> reports = TestUtils
-        srb.setCapacity(capacity).setScmUsed(used).
+            .createStorageReport(capacity, used, remaining, path, null, id, 1);
-            setRemaining(remaining).build();
         nodeManager.sendHeartbeat(datanodeDetails.getProtoBufMessage(),
-            nrb.addStorageReport(srb).build(), reportState);
+            TestUtils.createNodeReport(reports), reportState);
       }
 
       GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatProcessed(),

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java

@@ -281,9 +281,13 @@ public void testScmHeartbeatAfterRestart() throws Exception {
     conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
         100, TimeUnit.MILLISECONDS);
     DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
+    String dnId = datanodeDetails.getUuidString();
+    String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+    List<SCMStorageReport> reports =
+        TestUtils.createStorageReport(100, 10, 90, storagePath, null, dnId, 1);
     try (SCMNodeManager nodemanager = createNodeManager(conf)) {
       nodemanager.register(datanodeDetails.getProtoBufMessage(),
-          TestUtils.createNodeReport());
+          TestUtils.createNodeReport(reports));
       List<SCMCommand> command = nodemanager.sendHeartbeat(
           datanodeDetails.getProtoBufMessage(),
           null, reportState);
@@ -1012,14 +1016,14 @@ public void testScmStatsFromNodeReport() throws IOException,
       for (int x = 0; x < nodeCount; x++) {
         DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(
             nodeManager);
-
+        String dnId = datanodeDetails.getUuidString();
-        SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
+        long free = capacity - used;
-        SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+        String storagePath = testDir.getAbsolutePath() + "/" + dnId;
-        srb.setStorageUuid(UUID.randomUUID().toString());
+        List<SCMStorageReport> reports = TestUtils
-        srb.setCapacity(capacity).setScmUsed(used).
+            .createStorageReport(capacity, used, free, storagePath,
-            setRemaining(capacity - used).build();
+                null, dnId, 1);
         nodeManager.sendHeartbeat(datanodeDetails.getProtoBufMessage(),
-            nrb.addStorageReport(srb).build(), reportState);
+            TestUtils.createNodeReport(reports), reportState);
       }
       GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatProcessed(),
           100, 4 * 1000);
@@ -1055,21 +1059,21 @@ public void testScmNodeReportUpdate() throws IOException,
     conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
 
     try (SCMNodeManager nodeManager = createNodeManager(conf)) {
-      DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(
+      DatanodeDetails datanodeDetails =
-          nodeManager);
+          TestUtils.getDatanodeDetails(nodeManager);
       final long capacity = 2000;
       final long usedPerHeartbeat = 100;
-
+      String dnId = datanodeDetails.getUuidString();
       for (int x = 0; x < heartbeatCount; x++) {
-        SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
+        long scmUsed = x * usedPerHeartbeat;
-        SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+        long remaining = capacity - scmUsed;
-        srb.setStorageUuid(UUID.randomUUID().toString());
+        String storagePath = testDir.getAbsolutePath() + "/" + dnId;
-        srb.setCapacity(capacity).setScmUsed(x * usedPerHeartbeat)
+        List<SCMStorageReport> reports = TestUtils
-            .setRemaining(capacity - x * usedPerHeartbeat).build();
+            .createStorageReport(capacity, scmUsed, remaining, storagePath,
-        nrb.addStorageReport(srb);
+                null, dnId, 1);
 
-        nodeManager.sendHeartbeat(
+        nodeManager.sendHeartbeat(datanodeDetails.getProtoBufMessage(),
-            datanodeDetails.getProtoBufMessage(), nrb.build(), reportState);
+            TestUtils.createNodeReport(reports), reportState);
         Thread.sleep(100);
       }
 
@@ -1145,14 +1149,12 @@ public void testScmNodeReportUpdate() throws IOException,
       assertEquals(0, foundRemaining);
 
       // Send a new report to bring the dead node back to healthy
-      SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
+      String storagePath = testDir.getAbsolutePath() + "/" + dnId;
-      SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+      List<SCMStorageReport> reports = TestUtils
-      srb.setStorageUuid(UUID.randomUUID().toString());
+          .createStorageReport(capacity, expectedScmUsed, expectedRemaining,
-      srb.setCapacity(capacity).setScmUsed(expectedScmUsed)
+              storagePath, null, dnId, 1);
-          .setRemaining(expectedRemaining).build();
+      nodeManager.sendHeartbeat(datanodeDetails.getProtoBufMessage(),
-      nrb.addStorageReport(srb);
+          TestUtils.createNodeReport(reports), reportState);
-      nodeManager.sendHeartbeat(
-          datanodeDetails.getProtoBufMessage(), nrb.build(), reportState);
 
       // Wait up to 5 seconds so that the dead node becomes healthy
       // Verify usage info should be updated.

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeStorageStatMap.java

@@ -18,16 +18,17 @@
 package org.apache.hadoop.hdds.scm.node;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.TestUtils;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.*;
 import org.junit.rules.ExpectedException;
 
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
@@ -108,14 +109,17 @@ public void testProcessNodeReportCheckOneNode() throws SCMException {
     SCMNodeStorageStatMap map = new SCMNodeStorageStatMap(conf);
     map.insertNewDatanode(key, value);
     Assert.assertTrue(map.isKnownDatanode(key));
-    SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
+    String storageId = UUID.randomUUID().toString();
-    SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+    String path =
-    srb.setStorageUuid(UUID.randomUUID().toString());
+        GenericTestUtils.getRandomizedTempPath().concat("/" + storageId);
-    srb.setCapacity(value.getCapacity().get())
+    long capacity = value.getCapacity().get();
-        .setScmUsed(value.getScmUsed().get()).
+    long used = value.getScmUsed().get();
-        setRemaining(value.getRemaining().get()).build();
+    long remaining = value.getRemaining().get();
+    List<SCMStorageReport> reports = TestUtils
+        .createStorageReport(capacity, used, remaining, path, null, storageId,
+            1);
     SCMNodeStorageStatMap.NodeReportStatus status =
-        map.processNodeReport(key, nrb.addStorageReport(srb).build());
+        map.processNodeReport(key, TestUtils.createNodeReport(reports));
     Assert.assertEquals(status,
         SCMNodeStorageStatMap.NodeReportStatus.ALL_IS_WELL);
   }

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java

@@ -66,6 +66,7 @@
 
 import java.io.File;
 import java.net.InetSocketAddress;
+import java.util.List;
 import java.util.UUID;
 
 import static org.apache.hadoop.hdds.scm.TestUtils.getDatanodeDetails;
@@ -207,12 +208,12 @@ public void testGetVersionAssertRpcTimeOut() throws Exception {
   @Test
   public void testRegister() throws Exception {
     DatanodeDetails nodeToRegister = getDatanodeDetails();
-    try (EndpointStateMachine rpcEndPoint =
+    try (EndpointStateMachine rpcEndPoint = createEndpoint(
-             createEndpoint(
         SCMTestUtils.getConf(), serverAddress, 1000)) {
       SCMRegisteredCmdResponseProto responseProto = rpcEndPoint.getEndPoint()
-          .register(nodeToRegister.getProtoBufMessage(),
+          .register(nodeToRegister.getProtoBufMessage(), TestUtils
-              TestUtils.createNodeReport(),
+                  .createNodeReport(
+                      getStorageReports(nodeToRegister.getUuidString())),
               createContainerReport(10, nodeToRegister));
       Assert.assertNotNull(responseProto);
       Assert.assertEquals(nodeToRegister.getUuidString(),
@@ -220,11 +221,15 @@ public void testRegister() throws Exception {
       Assert.assertNotNull(responseProto.getClusterID());
       Assert.assertEquals(10, scmServerImpl.
           getContainerCountsForDatanode(nodeToRegister));
-      Assert.assertEquals(1, scmServerImpl.getNodeReportsCount(
+      Assert.assertEquals(1, scmServerImpl.getNodeReportsCount(nodeToRegister));
-          nodeToRegister));
     }
   }
 
+  private List<SCMStorageReport> getStorageReports(String id) {
+    String storagePath = testDir.getAbsolutePath() + "/" + id;
+    return TestUtils.createStorageReport(100, 10, 90, storagePath, null, id, 1);
+  }
+
   private EndpointStateMachine registerTaskHelper(InetSocketAddress scmAddress,
       int rpcTimeout, boolean clearDatanodeDetails) throws Exception {
     Configuration conf = SCMTestUtils.getConf();
@@ -234,7 +239,7 @@ private EndpointStateMachine registerTaskHelper(InetSocketAddress scmAddress,
     rpcEndPoint.setState(EndpointStateMachine.EndPointStates.REGISTER);
     OzoneContainer ozoneContainer = mock(OzoneContainer.class);
     when(ozoneContainer.getNodeReport()).thenReturn(TestUtils
-        .createNodeReport());
+        .createNodeReport(getStorageReports(UUID.randomUUID().toString())));
     when(ozoneContainer.getContainerReport()).thenReturn(
         createContainerReport(10, null));
     RegisterEndpointTask endpointTask =
@@ -297,14 +302,11 @@ public void testHeartbeat() throws Exception {
     try (EndpointStateMachine rpcEndPoint =
              createEndpoint(SCMTestUtils.getConf(),
                  serverAddress, 1000)) {
-      SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
+      String storageId = UUID.randomUUID().toString();
-      SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
-      srb.setStorageUuid(UUID.randomUUID().toString());
-      srb.setCapacity(2000).setScmUsed(500).setRemaining(1500).build();
-      nrb.addStorageReport(srb);
       SCMHeartbeatResponseProto responseProto = rpcEndPoint.getEndPoint()
-          .sendHeartbeat(
+          .sendHeartbeat(dataNode.getProtoBufMessage(),
-              dataNode.getProtoBufMessage(), nrb.build(), defaultReportState);
+              TestUtils.createNodeReport(getStorageReports(storageId)),
+              defaultReportState);
       Assert.assertNotNull(responseProto);
       Assert.assertEquals(0, responseProto.getCommandsCount());
     }