HDFS-10715. NPE when applying AvailableSpaceBlockPlacementPolicy. Contributed by Guangbin Zhu.

(cherry picked from commit ef432579a7)
(cherry picked from commit 6863866127)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java

@@ -76,13 +76,17 @@ public class AvailableSpaceBlockPlacementPolicy extends
         (DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode);
     DatanodeDescriptor b =
         (DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode);
-    int ret = compareDataNode(a, b);
+    if (a != null && b != null){
-    if (ret == 0) {
+      int ret = compareDataNode(a, b);
-      return a;
+      if (ret == 0) {
-    } else if (ret < 0) {
+        return a;
-      return (RAND.nextInt(100) < balancedPreference) ? a : b;
+      } else if (ret < 0) {
+        return (RAND.nextInt(100) < balancedPreference) ? a : b;
+      } else {
+        return (RAND.nextInt(100) < balancedPreference) ? b : a;
+      }
     } else {
-      return (RAND.nextInt(100) < balancedPreference) ? b : a;
+      return a == null ? b : a;
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -30,6 +32,7 @@ import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.Node;
 import org.apache.hadoop.test.PathUtils;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -158,6 +161,21 @@ public class TestAvailableSpaceBlockPlacementPolicy {
     Assert.assertTrue(possibility < 0.55);
   }
 
+  @Test
+  public void testChooseDataNode() {
+    try {
+      Collection<Node> allNodes = new ArrayList<>(dataNodes.length);
+      Collections.addAll(allNodes, dataNodes);
+      if (placementPolicy instanceof AvailableSpaceBlockPlacementPolicy){
+        // exclude all datanodes when chooseDataNode, no NPE should be thrown
+        ((AvailableSpaceBlockPlacementPolicy)placementPolicy)
+                .chooseDataNode("~", allNodes);
+      }
+    }catch (NullPointerException npe){
+      Assert.fail("NPE should not be thrown");
+    }
+  }
+
   @AfterClass
   public static void teardownCluster() {
     if (namenode != null) {