MAPREDUCE-6715. Fix Several Unsafe Practices (Contributed by Yufei Gu via Daniel Templeton)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java

@@ -136,7 +136,12 @@ class CleanupQueue {
             LOG.debug("DELETED " + context.fullPath);
           }
         } catch (InterruptedException t) {
-          LOG.warn("Interrupted deletion of " + context.fullPath);
+          if (context == null) {
+            LOG.warn("Interrupted deletion of an invalid path: Path deletion "
+                + "context is null.");
+          } else {
+            LOG.warn("Interrupted deletion of " + context.fullPath);
+          }
           return;
         } catch (Exception e) {
           LOG.warn("Error deleting path " + context.fullPath + ": " + e);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java

@@ -411,8 +411,14 @@ public class MapTask extends Task {
         LOG.warn(msg, e);
       }
     }
-    throw new IOException("Initialization of all the collectors failed. " +
+
-      "Error in last collector was :" + lastException.getMessage(), lastException);
+    if (lastException != null) {
+      throw new IOException("Initialization of all the collectors failed. " +
+          "Error in last collector was:" + lastException.toString(),
+          lastException);
+    } else {
+      throw new IOException("Initialization of all the collectors failed.");
+    }
   }
 
   @SuppressWarnings("unchecked")

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java

@@ -113,19 +113,18 @@ public class TextOutputFormat<K, V> extends FileOutputFormat<K, V> {
     if (isCompressed) {
       Class<? extends CompressionCodec> codecClass = 
         getOutputCompressorClass(job, GzipCodec.class);
-      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
+      codec = ReflectionUtils.newInstance(codecClass, conf);
       extension = codec.getDefaultExtension();
     }
     Path file = getDefaultWorkFile(job, extension);
     FileSystem fs = file.getFileSystem(conf);
-    if (!isCompressed) {
+    FSDataOutputStream fileOut = fs.create(file, false);
-      FSDataOutputStream fileOut = fs.create(file, false);
+    if (isCompressed) {
-      return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);
+      return new LineRecordWriter<>(
+          new DataOutputStream(codec.createOutputStream(fileOut)),
+          keyValueSeparator);
     } else {
-      FSDataOutputStream fileOut = fs.create(file, false);
+      return new LineRecordWriter<>(fileOut, keyValueSeparator);
-      return new LineRecordWriter<K, V>(new DataOutputStream
-                                        (codec.createOutputStream(fileOut)),
-                                        keyValueSeparator);
     }
   }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java

@@ -433,25 +433,29 @@ public class ShuffleSchedulerImpl<K,V> implements ShuffleScheduler<K,V> {
 
 
   public synchronized MapHost getHost() throws InterruptedException {
-      while(pendingHosts.isEmpty()) {
+    while(pendingHosts.isEmpty()) {
-        wait();
+      wait();
-      }
+    }
 
-      MapHost host = null;
+    Iterator<MapHost> iter = pendingHosts.iterator();
-      Iterator<MapHost> iter = pendingHosts.iterator();
+    // Safe to take one because we know pendingHosts isn't empty
-      int numToPick = random.nextInt(pendingHosts.size());
+    MapHost host = iter.next();
-      for (int i=0; i <= numToPick; ++i) {
+    int numToPick = random.nextInt(pendingHosts.size());
-        host = iter.next();
+    for (int i = 0; i < numToPick; ++i) {
-      }
+      host = iter.next();
+    }
 
-      pendingHosts.remove(host);
+    pendingHosts.remove(host);
-      host.markBusy();
+    host.markBusy();
 
-      LOG.debug("Assigning " + host + " with " + host.getNumKnownMapOutputs() +
+    if (LOG.isDebugEnabled()) {
-               " to " + Thread.currentThread().getName());
+      LOG.debug(
-      SHUFFLE_START.set(Time.monotonicNow());
+          "Assigning " + host + " with " + host.getNumKnownMapOutputs() + " to "
+              + Thread.currentThread().getName());
+    }
+    SHUFFLE_START.set(Time.monotonicNow());
 
-      return host;
+    return host;
   }
 
   public synchronized List<TaskAttemptID> getMapsForHost(MapHost host) {
@@ -477,8 +481,10 @@ public class ShuffleSchedulerImpl<K,V> implements ShuffleScheduler<K,V> {
         host.addKnownMap(id);
       }
     }
-    LOG.debug("assigned " + includedMaps + " of " + totalSize + " to " +
+    if (LOG.isDebugEnabled()) {
-             host + " to " + Thread.currentThread().getName());
+      LOG.debug("assigned " + includedMaps + " of " + totalSize + " to " + host
+          + " to " + Thread.currentThread().getName());
+    }
     return result;
   }
 

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java

@@ -153,6 +153,11 @@ public class Pentomino {
           break;
         }
       }
+
+      if (piece == null) {
+        continue;
+      }
+
       // for each point where the piece was placed, mark it with the piece name
       for(ColumnName item: row) {
         if (item instanceof Point) {

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java

@@ -73,14 +73,14 @@ class TeraScheduler {
 
   List<String> readFile(String filename) throws IOException {
     List<String> result = new ArrayList<String>(10000);
-    BufferedReader in = new BufferedReader(
+    try (BufferedReader in = new BufferedReader(
-        new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8));
+        new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8))) {
-    String line = in.readLine();
+      String line = in.readLine();
-    while (line != null) {
+      while (line != null) {
-      result.add(line);
+        result.add(line);
-      line = in.readLine();
+        line = in.readLine();
+      }
     }
-    in.close();
     return result;
   }